@gscdump/engine 0.38.2 → 0.40.1

package/dist/_chunks/resolver.mjs

@@ -455,7 +455,6 @@ function createParquetResolverAdapter(options = {}) {
 	return createResolverAdapter({
 		...PG_BASE_CONFIG,
 		tableLabel: "parquet-resolver-adapter",
-		canonicalFallback: options.canonicalFallback ?? false,
 		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
 		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`),
 		queryDimTableRef: () => sql.raw("read_parquet({{QUERY_DIM}}, union_by_name = true) AS \"query_dim\"")
@@ -468,7 +467,6 @@ function createIcebergResolverAdapter(options = {}) {
 		includeSiteId: true,
 		includeSearchType: true,
 		tableLabel: "iceberg-resolver-adapter",
-		canonicalFallback: options.canonicalFallback ?? false,
 		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
 		tableRef: (tk) => sql.raw(`"${tk}"`),
 		queryDimTableRef: () => sql.raw("\"query_dim\"")
@@ -481,7 +479,6 @@ function createR2SqlResolverAdapter(options = {}) {
 		includeSiteId: true,
 		includeSearchType: true,
 		tableLabel: "r2-sql-resolver-adapter",
-		canonicalFallback: options.canonicalFallback ?? false,
 		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
 		capabilities: {
 			regex: false,
@@ -822,65 +819,67 @@ function buildExtrasQueries(state, options) {
 function mergeExtras(rows, extrasResults) {
 	if (extrasResults.length === 0) return rows;
 	const lookups = [];
+	const parseVariants = (raw) => {
+		if (raw.length === 0) return [];
+		const encoded = raw.split("||");
+		const variants = [];
+		for (const value of encoded) {
+			if (value.length === 0) continue;
+			const parts = value.split(":::");
+			variants.push({
+				query: parts[0],
+				clicks: Number(parts[1] || 0),
+				impressions: Number(parts[2] || 0),
+				position: Number(parts[3] || 0)
+			});
+		}
+		return variants;
+	};
 	for (const { key, results } of extrasResults) {
 		if (key === "canonicalExtras") {
-			const variantCountMap = /* @__PURE__ */ new Map();
-			const variantsMap = /* @__PURE__ */ new Map();
-			const canonicalNameMap = /* @__PURE__ */ new Map();
+			const map = /* @__PURE__ */ new Map();
 			for (const r of results) {
 				const jk = String(r.joinKey);
-				variantCountMap.set(jk, r.variantCount);
-				canonicalNameMap.set(jk, r.canonicalName);
 				const raw = r.variants;
-				variantsMap.set(jk, typeof raw === "string" ? raw.split("||").filter(Boolean).map((v) => {
-					const parts = v.split(":::");
-					return {
-						query: parts[0],
-						clicks: Number(parts[1] || 0),
-						impressions: Number(parts[2] || 0),
-						position: Number(parts[3] || 0)
-					};
-				}) : []);
+				map.set(jk, {
+					variantCount: r.variantCount,
+					variants: typeof raw === "string" ? parseVariants(raw) : [],
+					canonicalName: r.canonicalName
+				});
 			}
 			lookups.push({
-				key: "variantCount",
-				map: variantCountMap
-			});
-			lookups.push({
-				key: "variants",
-				map: variantsMap
-			});
-			lookups.push({
-				key: "canonicalName",
-				map: canonicalNameMap
+				kind: "canonical",
+				map
 			});
 			continue;
 		}
-		const filtered = results.filter((r) => r.rn === void 0 || r.rn === 1);
 		const map = /* @__PURE__ */ new Map();
-		for (const r of filtered) {
+		for (const r of results) {
+			if (r.rn !== void 0 && r.rn !== 1) continue;
 			let val = r[key];
-			if (key === "variants" && typeof val === "string") val = val.split("||").filter(Boolean).map((v) => {
-				const parts = v.split(":::");
-				return {
-					query: parts[0],
-					clicks: Number(parts[1] || 0),
-					impressions: Number(parts[2] || 0),
-					position: Number(parts[3] || 0)
-				};
-			});
+			if (key === "variants" && typeof val === "string") val = parseVariants(val);
 			map.set(String(r.joinKey), val);
 		}
 		lookups.push({
+			kind: "generic",
 			key,
 			map
 		});
 	}
 	return rows.map((row) => {
 		const enriched = { ...row };
-		for (const { key, map } of lookups) {
-			let joinValue;
-			if (key === "variantCount" || key === "variants" || key === "canonicalName") joinValue = String(row.queryCanonical ?? row.query_canonical ?? "");
+		for (const lookup of lookups) {
+			if (lookup.kind === "canonical") {
+				const joinValue = String(row.queryCanonical ?? row.query_canonical ?? "");
+				const extra = joinValue ? lookup.map.get(joinValue) : void 0;
+				enriched.variantCount = extra?.variantCount ?? null;
+				enriched.variants = extra?.variants ?? [];
+				enriched.canonicalName = extra?.canonicalName ?? null;
+				if (enriched.canonicalName) enriched.queryCanonical = enriched.canonicalName;
+				continue;
+			}
+			const { key, map } = lookup;
+			const joinValue = key === "variantCount" || key === "variants" || key === "canonicalName" ? String(row.queryCanonical ?? row.query_canonical ?? "") : void 0;
 			enriched[key] = (joinValue && map.get(joinValue)) ?? (key === "variants" ? [] : null);
 			if (key === "canonicalName" && enriched[key]) enriched.queryCanonical = enriched[key];
 		}
@@ -955,7 +954,10 @@ function matchesMetricFilter(row, filter) {
 	}
 }
 function matchesTopLevelPage(row) {
-	return (normalizeUrl(dimensionValue(row, "page")).match(/\//g)?.length ?? 0) <= 1;
+	const path = normalizeUrl(dimensionValue(row, "page"));
+	let slashes = 0;
+	for (let index = 0; index < path.length; index++) if (path.charCodeAt(index) === 47 && ++slashes > 1) return false;
+	return true;
 }
 var QuerySourceCoverageError = class extends Error {
 	fallback;

package/dist/_chunks/schema.d.mts

@@ -2115,7 +2115,6 @@ declare const TABLE_METADATA: Record<TableName, {
 }>;
 declare const SCHEMAS: Record<TableName, TableSchema>;
 declare function currentSchemaVersion(table: TableName): number;
-declare function schemaFor(table: TableName): TableSchema;
 /**
  * DATE column names for a table. The single schema-derived source every read
  * path uses to build the legacy-VARCHAR date canonicalization (see
@@ -2146,4 +2145,4 @@ declare function naturalKeyColumns(table: TableName): readonly string[];
  */
 declare function dedupeByNaturalKey(table: TableName, rows: readonly Row[]): Row[];
 declare function dimensionToColumn(dim: string, _table: TableName): string;
-export { type ColumnDef$1 as ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+export { type ColumnDef$1 as ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/_chunks/schema.mjs

@@ -219,9 +219,6 @@ const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(
 function currentSchemaVersion(table) {
 	return SCHEMAS[table].version;
 }
-function schemaFor(table) {
-	return SCHEMAS[table];
-}
 function dateColumnsFor(table) {
 	return SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
 }
@@ -259,4 +256,4 @@ function dimensionToColumn(dim, _table) {
 	if (dim === "queryCanonical") return "query_canonical";
 	return dim;
 }
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/_chunks/source.mjs

@@ -137,21 +137,7 @@ async function runAnalyzerWithEngine(deps, ctx, params, registry) {
 		searchType: params.searchType ?? "web"
 	}), params, registry);
 }
-function typedQuery(state) {
-	return { state };
-}
-function isTypedQuery(value) {
-	return "state" in value;
-}
-async function queryRows(source, query) {
-	const state = isTypedQuery(query) ? query.state : query;
+async function queryRows(source, state) {
 	return await source.queryRows(state);
 }
-async function queryComparisonRows(source, current, previous) {
-	const [currentRows, previousRows] = await Promise.all([queryRows(source, current), queryRows(source, previous)]);
-	return {
-		current: currentRows,
-		previous: previousRows
-	};
-}
-export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };
+export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine };

package/dist/adapters/filesystem.mjs

@@ -2,31 +2,71 @@ import { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, m
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
-import { mkdir, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { mkdir, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
 import { lock } from "proper-lockfile";
 function createFilesystemDataSource(opts) {
 	const root = resolve(opts.rootDir);
+	const readyDirectories = /* @__PURE__ */ new Map();
 	function pathFor(key) {
 		const resolved = resolve(root, key);
 		if (!resolved.startsWith(`${root}/`) && resolved !== root) throw new Error(`path escapes root: ${key}`);
 		return resolved;
 	}
+	async function ensureDirectory(path) {
+		let pending = readyDirectories.get(path);
+		if (!pending) {
+			pending = mkdir(path, { recursive: true }).then(() => void 0);
+			readyDirectories.set(path, pending);
+			pending.catch(() => readyDirectories.delete(path));
+		}
+		await pending;
+	}
 	return {
 		async read(key, range, signal) {
-			const bytes = await readFile(pathFor(key), { signal });
-			if (!range) return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-			const sliced = bytes.subarray(range.offset, range.offset + range.length);
-			return new Uint8Array(sliced.buffer, sliced.byteOffset, sliced.byteLength);
+			const path = pathFor(key);
+			if (!range) {
+				const bytes = await readFile(path, { signal });
+				return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+			}
+			signal?.throwIfAborted();
+			const handle = await open(path, "r");
+			try {
+				const bytes = Buffer.allocUnsafe(range.length);
+				let totalRead = 0;
+				while (totalRead < range.length) {
+					signal?.throwIfAborted();
+					const { bytesRead } = await handle.read(bytes, totalRead, range.length - totalRead, range.offset + totalRead);
+					if (bytesRead === 0) break;
+					totalRead += bytesRead;
+				}
+				return new Uint8Array(bytes.buffer, bytes.byteOffset, totalRead);
+			} finally {
+				await handle.close();
+			}
 		},
 		async write(key, bytes) {
 			const path = pathFor(key);
-			await mkdir(dirname(path), { recursive: true });
-			await writeFile(path, Buffer.from(bytes));
+			const dir = dirname(path);
+			await ensureDirectory(dir);
+			try {
+				await writeFile(path, bytes);
+			} catch (error) {
+				if (error.code !== "ENOENT") throw error;
+				readyDirectories.delete(dir);
+				await ensureDirectory(dir);
+				await writeFile(path, bytes);
+			}
 		},
 		async delete(keys) {
-			await Promise.all(keys.map(async (k) => {
-				await rm(pathFor(k), { force: true });
-			}));
+			let next = 0;
+			async function worker() {
+				while (true) {
+					const index = next++;
+					if (index >= keys.length) return;
+					await rm(pathFor(keys[index]), { force: true });
+				}
+			}
+			await Promise.all(Array.from({ length: Math.min(32, keys.length) }, worker));
 		},
 		async list(prefix) {
 			const full = pathFor(prefix);
@@ -61,14 +101,23 @@ async function* walkStream(dir) {
 	}
 }
 async function walk(dir, out) {
-	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
-		if (err.code === "ENOENT") return [];
-		throw err;
-	});
-	for (const entry of entries) {
-		const p = join(dir, String(entry.name));
-		if (entry.isDirectory()) await walk(p, out);
-		else out.push(p);
+	const pending = [dir];
+	let next = 0;
+	while (next < pending.length) {
+		const batch = pending.slice(next, next + 32);
+		next += batch.length;
+		const listings = await Promise.all(batch.map(async (current) => ({
+			current,
+			entries: await readdir(current, { withFileTypes: true }).catch((err) => {
+				if (err.code === "ENOENT") return [];
+				throw err;
+			})
+		})));
+		for (const { current, entries } of listings) for (const entry of entries) {
+			const p = join(current, String(entry.name));
+			if (entry.isDirectory()) pending.push(p);
+			else out.push(p);
+		}
 	}
 }
 function lockFileFor(locksDir, scope) {
@@ -77,6 +126,16 @@ function lockFileFor(locksDir, scope) {
 function createFilesystemManifestStore(opts) {
 	const manifestPath = resolve(opts.path);
 	const locksDir = join(dirname(manifestPath), "locks");
+	const readyDirectories = /* @__PURE__ */ new Map();
+	async function ensureDirectory(path) {
+		let pending = readyDirectories.get(path);
+		if (!pending) {
+			pending = mkdir(path, { recursive: true }).then(() => void 0);
+			readyDirectories.set(path, pending);
+			pending.catch(() => readyDirectories.delete(path));
+		}
+		await pending;
+	}
 	async function load() {
 		const content = await readFile(manifestPath, "utf8").catch((err) => {
 			if (err.code === "ENOENT") return null;
@@ -91,9 +150,16 @@ function createFilesystemManifestStore(opts) {
 		return parsed;
 	}
 	async function save(data) {
-		await mkdir(dirname(manifestPath), { recursive: true });
+		const manifestDir = dirname(manifestPath);
+		await ensureDirectory(manifestDir);
 		const tmp = `${manifestPath}.${randomBytes(6).toString("hex")}.tmp`;
-		await writeFile(tmp, JSON.stringify(data), "utf8");
+		const content = JSON.stringify(data);
+		await writeFile(tmp, content, "utf8").catch(async (error) => {
+			if (error.code !== "ENOENT") throw error;
+			readyDirectories.delete(manifestDir);
+			await ensureDirectory(manifestDir);
+			await writeFile(tmp, content, "utf8");
+		});
 		await rename(tmp, manifestPath).catch(async (err) => {
 			try {
 				await unlink(tmp);
@@ -175,9 +241,14 @@ function createFilesystemManifestStore(opts) {
 			});
 		},
 		async withLock(scope, fn) {
-			await mkdir(locksDir, { recursive: true });
+			await ensureDirectory(locksDir);
 			const path = lockFileFor(locksDir, scope);
-			await writeFile(path, "", { flag: "a" });
+			await writeFile(path, "", { flag: "a" }).catch(async (error) => {
+				if (error.code !== "ENOENT") throw error;
+				readyDirectories.delete(locksDir);
+				await ensureDirectory(locksDir);
+				await writeFile(path, "", { flag: "a" });
+			});
 			const release = await lock(path, {
 				realpath: false,
 				stale: 3e4,
@@ -244,24 +315,22 @@ async function filesystemStats(rootDir) {
 	const keys = [];
 	await walkForStats(resolve(rootDir), keys);
 	let bytes = 0;
-	for (const k of keys) {
-		const s = await stat(k);
-		bytes += s.size;
+	let next = 0;
+	async function worker() {
+		while (true) {
+			const index = next++;
+			if (index >= keys.length) return;
+			const s = await stat(keys[index]);
+			bytes += s.size;
+		}
 	}
+	await Promise.all(Array.from({ length: Math.min(32, keys.length) }, worker));
 	return {
 		files: keys.length,
 		bytes
 	};
 }
 async function walkForStats(dir, out) {
-	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
-		if (err.code === "ENOENT") return [];
-		throw err;
-	});
-	for (const entry of entries) {
-		const p = join(dir, String(entry.name));
-		if (entry.isDirectory()) await walkForStats(p, out);
-		else out.push(p);
-	}
+	await walk(dir, out);
 }
 export { createFilesystemDataSource, createFilesystemManifestStore, filesystemStats };

package/dist/adapters/hyparquet.d.mts

@@ -45,6 +45,11 @@ interface HyparquetCodecOptions {
    * hyparquet to avoid WASM linear-memory growth. Defaults to hyparquet.
    */
   readRows?: (ctx: CodecCtx, key: string, dataSource: DataSource) => Promise<Row[]>;
+  /**
+   * Maximum input-file reads held in flight during compaction. Defaults to 8;
+   * lower it for unusually large files to trade latency for peak JS memory.
+   */
+  compactionReadConcurrency?: number;
 }
 declare function createHyparquetCodec(options?: HyparquetCodecOptions): ParquetCodec;
 export { DecodeParquetOptions, EncodeFlexOptions, HyparquetCodecOptions, createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };

package/dist/adapters/hyparquet.mjs

@@ -2,6 +2,8 @@ import { SCHEMAS, TABLE_METADATA } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "../_chunks/libs/hyparquet.mjs";
 import { ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
 const ROW_GROUP_SIZE = 25e3;
+const DEFAULT_COMPACTION_READ_CONCURRENCY = 8;
+const MAX_COMPACTION_READ_CONCURRENCY = 32;
 function basicTypeFor(colType) {
 	if (colType === "VARCHAR") return "STRING";
 	if (colType === "BIGINT") return "INT64";
@@ -120,7 +122,13 @@ function sortRowsByClusterKey(table, rows) {
 	return copy;
 }
 function naturalKeyFor(table, row) {
-	return TABLE_METADATA[table].sortKey.map((col) => `${row[col] ?? ""}`).join("\0");
+	const key = TABLE_METADATA[table].sortKey;
+	let value = "";
+	for (let index = 0; index < key.length; index++) {
+		if (index > 0) value += "\0";
+		value += `${row[key[index]] ?? ""}`;
+	}
+	return value;
 }
 function encodeOrderedRows(rows, columns, rowGroupSize) {
 	const schema = buildWriteSchema(columns);
@@ -204,6 +212,11 @@ function normalizeDecodedDates(rows) {
 	return rows;
 }
 function createHyparquetCodec(options = {}) {
+	const readRows = options.readRows ?? (async (_ctx, key, dataSource) => {
+		return decodeParquetToRows(await dataSource.read(key));
+	});
+	const requestedConcurrency = options.compactionReadConcurrency;
+	const compactionReadConcurrency = typeof requestedConcurrency === "number" && Number.isFinite(requestedConcurrency) ? Math.max(1, Math.min(MAX_COMPACTION_READ_CONCURRENCY, Math.floor(requestedConcurrency))) : DEFAULT_COMPACTION_READ_CONCURRENCY;
 	return {
 		async writeRows(ctx, rows, key, dataSource) {
 			const bytes = encodeRowsToParquet(ctx.table, rows);
@@ -213,9 +226,7 @@ function createHyparquetCodec(options = {}) {
 				rowCount: rows.length
 			};
 		},
-		readRows: options.readRows ?? (async (_ctx, key, dataSource) => {
-			return decodeParquetToRows(await dataSource.read(key));
-		}),
+		readRows,
 		async compactRows(ctx, inputKeys, outputKey, dataSource) {
 			if (inputKeys.length === 0) {
 				const bytes = encodeRowsToParquet(ctx.table, []);
@@ -226,9 +237,13 @@ function createHyparquetCodec(options = {}) {
 				};
 			}
 			const byNaturalKey = /* @__PURE__ */ new Map();
-			for (const key of inputKeys) {
-				const rows = await decodeParquetToRows(await dataSource.read(key));
-				for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
+			for (let offset = 0; offset < inputKeys.length; offset += compactionReadConcurrency) {
+				const batch = inputKeys.slice(offset, offset + compactionReadConcurrency);
+				const inputs = await Promise.all(batch.map((key) => dataSource.read(key)));
+				for (const input of inputs) {
+					const rows = await decodeParquetToRows(input);
+					for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
+				}
 			}
 			const rows = [...byNaturalKey.values()];
 			const bytes = encodeRowsToParquet(ctx.table, rows);

package/dist/adapters/r2-manifest.mjs

@@ -6,6 +6,7 @@ const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
 const CAS_BACKOFF_CAP_MS = 250;
 const SHARD_IO_CONCURRENCY = 8;
+const R2_DELETE_BATCH_SIZE = 1e3;
 async function casBackoff(attempt) {
 	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
 	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
@@ -331,15 +332,8 @@ function createR2ManifestStore(opts) {
 		},
 		async purgeTenant(filter) {
 			if (filter.userId !== userId) throw new Error(`purgeTenant: store is scoped to userId=${userId}, got ${filter.userId}`);
-			const shards = await shardsForFilter({ siteId: filter.siteId });
-			let entriesRemoved = 0;
-			let watermarksRemoved = 0;
-			let syncStatesRemoved = 0;
-			for (const { siteId, table } of shards) {
+			const purged = await mapWithConcurrency(await shardsForFilter({ siteId: filter.siteId }), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				entriesRemoved += snapshot.entries.length;
-				watermarksRemoved += snapshot.watermarks.length;
-				syncStatesRemoved += snapshot.syncStates.length;
 				const prefix = shardPrefix(userId, siteId, table);
 				const keys = [];
 				let cursor;
@@ -352,12 +346,17 @@ function createR2ManifestStore(opts) {
 					for (const obj of res.objects) keys.push(obj.key);
 					cursor = res.truncated ? res.cursor : void 0;
 				} while (cursor);
-				if (keys.length > 0) await bucket.delete(keys);
-			}
+				for (let offset = 0; offset < keys.length; offset += R2_DELETE_BATCH_SIZE) await bucket.delete(keys.slice(offset, offset + R2_DELETE_BATCH_SIZE));
+				return {
+					entriesRemoved: snapshot.entries.length,
+					watermarksRemoved: snapshot.watermarks.length,
+					syncStatesRemoved: snapshot.syncStates.length
+				};
+			});
 			return {
-				entriesRemoved,
-				watermarksRemoved,
-				syncStatesRemoved
+				entriesRemoved: purged.reduce((sum, result) => sum + result.entriesRemoved, 0),
+				watermarksRemoved: purged.reduce((sum, result) => sum + result.watermarksRemoved, 0),
+				syncStatesRemoved: purged.reduce((sum, result) => sum + result.syncStatesRemoved, 0)
 			};
 		}
 	};

package/dist/adapters/r2.mjs

@@ -1,5 +1,7 @@
 const BUCKET_NAME_RE = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/;
 const KEY_RE = /^[\w./=-]+$/;
+const DELETE_CHUNK_SIZE = 1e3;
+const DELETE_CONCURRENCY = 4;
 function assertKey(key) {
 	if (!KEY_RE.test(key)) throw new Error(`createR2DataSource: refusing unsafe key ${JSON.stringify(key)}`);
 }
@@ -22,8 +24,17 @@ function createR2DataSource(options) {
 		},
 		async delete(keys) {
 			if (keys.length === 0) return;
-			const CHUNK = 1e3;
-			for (let i = 0; i < keys.length; i += CHUNK) await bucket.delete(keys.slice(i, i + CHUNK));
+			let next = 0;
+			async function worker() {
+				while (true) {
+					const offset = next;
+					next += DELETE_CHUNK_SIZE;
+					if (offset >= keys.length) return;
+					await bucket.delete(keys.slice(offset, offset + DELETE_CHUNK_SIZE));
+				}
+			}
+			const chunkCount = Math.ceil(keys.length / DELETE_CHUNK_SIZE);
+			await Promise.all(Array.from({ length: Math.min(DELETE_CONCURRENCY, chunkCount) }, worker));
 		},
 		async list(prefix) {
 			const out = [];

package/dist/errors.d.mts

@@ -1,2 +1,2 @@
-import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, formatEngineError, isEngineError } from "./_chunks/errors.mjs";
-export { EngineError, EngineErrorKind, engineErrorToException, engineErrors, formatEngineError, isEngineError };
+import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, isEngineError } from "./_chunks/errors.mjs";
+export { EngineError, EngineErrorKind, engineErrorToException, engineErrors, isEngineError };

package/dist/errors.mjs

@@ -190,13 +190,10 @@ const ENGINE_ERROR_KINDS = /* @__PURE__ */ new Set([
 function isEngineError(value) {
 	return typeof value === "object" && value !== null && ENGINE_ERROR_KINDS.has(value.kind) && typeof value.message === "string";
 }
-function formatEngineError(error) {
-	return error.message;
-}
 function engineErrorToException(error) {
 	const exception = new Error(error.message);
 	if ("cause" in error && error.cause !== void 0) exception.cause = error.cause;
 	exception.engineError = error;
 	return exception;
 }
-export { engineErrorToException, engineErrors, formatEngineError, isEngineError };
+export { engineErrorToException, engineErrors, isEngineError };

package/dist/iceberg/index.mjs

@@ -137,8 +137,12 @@ function dedupeByIdentity(table, records) {
 	const key = gscDataset(table, "int").tableSpec.identityColumns;
 	const seen = /* @__PURE__ */ new Map();
 	for (const rec of records) {
-		const k = key.map((col) => `${rec[col] ?? ""}`).join("\0");
-		seen.set(k, rec);
+		let identity = "";
+		for (let index = 0; index < key.length; index++) {
+			if (index > 0) identity += "\0";
+			identity += `${rec[key[index]] ?? ""}`;
+		}
+		seen.set(identity, rec);
 	}
 	return seen.size === records.length ? records : [...seen.values()];
 }

package/dist/index.d.mts

@@ -1,7 +1,7 @@
 import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
 import { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
 import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
-import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, formatEngineError, isEngineError } from "./_chunks/errors.mjs";
+import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, isEngineError } from "./_chunks/errors.mjs";
 import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
@@ -109,7 +109,6 @@ interface CreateIngestAccumulatorOptions extends RowAccumulatorOptions {
   ctx: IngestAccumulatorCtx;
   hooks: IngestAccumulatorHooks;
 }
-declare function createNoopIngestAccumulator(): IngestAccumulator;
 declare function createIngestAccumulator(opts: CreateIngestAccumulatorOptions): IngestAccumulator;
 declare function dayPartition(date: string): string;
 /**
@@ -209,4 +208,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };