@gscdump/engine 1.4.0 → 1.4.2

package/dist/_chunks/engine.mjs

@@ -1,731 +0,0 @@
-import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
-import { coerceRows } from "./coerce.mjs";
-import { SCHEMAS, TABLE_METADATA, currentSchemaVersion, dateColumnsFor, dedupeByNaturalKey } from "./schema.mjs";
-import { compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, splitOverlappingTiers, substituteNamedFiles } from "./parquet-plan.mjs";
-import { dateReplaceClause as dateReplaceClause$1 } from "../sql-fragments.mjs";
-import { sqlEscape } from "../sql-bind.mjs";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
-import { buildLogicalPlan } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump/normalize";
-const DEFAULT_BUFFER_READ_CONCURRENCY = 16;
-const MAX_BUFFER_READ_CONCURRENCY = 64;
-function bufferReadConcurrency(value, fileCount) {
-	return Math.max(1, Math.min(fileCount || 1, typeof value === "number" && Number.isFinite(value) ? Math.floor(value) : DEFAULT_BUFFER_READ_CONCURRENCY, MAX_BUFFER_READ_CONCURRENCY));
-}
-async function registerBufferedFiles(db, files, dataSource, registered, signal, maxConcurrency) {
-	const batchSize = bufferReadConcurrency(maxConcurrency, files.length);
-	for (let i = 0; i < files.length; i += batchSize) {
-		signal?.throwIfAborted();
-		const batch = files.slice(i, i + batchSize);
-		const buffers = await Promise.all(batch.map((file) => dataSource.read(file.key, void 0, signal)));
-		for (let j = 0; j < batch.length; j++) {
-			signal?.throwIfAborted();
-			const file = batch[j];
-			await db.registerFileBuffer(file.name, buffers[j]);
-			registered.push(file.name);
-		}
-	}
-}
-async function encodeBytes(db, table, rows) {
-	const inName = db.makeTempPath("json");
-	const outName = db.makeTempPath("parquet");
-	const jsonBytes = encodeJsonBigintSafe(coerceRows(rows));
-	const registered = [];
-	await db.registerFileBuffer(inName, jsonBytes);
-	registered.push(inName);
-	try {
-		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`;
-		await db.query(sql);
-		registered.push(outName);
-		return await db.copyFileToBuffer(outName);
-	} finally {
-		await db.dropFiles(registered);
-	}
-}
-async function decodeBytes(db, bytes, table) {
-	const name = db.makeTempPath("parquet");
-	await db.registerFileBuffer(name, bytes);
-	try {
-		return await db.query(`SELECT * ${dateReplaceClause(table)} FROM read_parquet('${sqlEscape(name)}')`);
-	} finally {
-		await db.dropFiles([name]);
-	}
-}
-function createDuckDBCodec(factory, options = {}) {
-	return {
-		async writeRows(ctx, rows, key, dataSource) {
-			const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
-			await dataSource.write(key, bytes);
-			return {
-				bytes: bytes.byteLength,
-				rowCount: rows.length
-			};
-		},
-		async readRows(ctx, key, dataSource) {
-			return decodeBytes(await factory.getDuckDB(), await dataSource.read(key), ctx.table);
-		},
-		async compactRows(ctx, inputKeys, outputKey, dataSource) {
-			const db = await factory.getDuckDB();
-			if (inputKeys.length === 0) {
-				const bytes = await encodeBytes(db, ctx.table, []);
-				await dataSource.write(outputKey, bytes);
-				return {
-					bytes: bytes.byteLength,
-					rowCount: 0
-				};
-			}
-			const inputUris = inputKeys.map((k) => dataSource.uri?.(k));
-			if (inputUris.every((u) => u !== void 0)) {
-				const outName = db.makeTempPath("parquet");
-				const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
-				try {
-					await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
-					const bytes = await db.copyFileToBuffer(outName);
-					const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
-					const rowCount = Number(countRows[0]?.n ?? 0);
-					await dataSource.write(outputKey, bytes);
-					return {
-						bytes: bytes.byteLength,
-						rowCount
-					};
-				} finally {
-					await db.dropFiles([outName]);
-				}
-			}
-			const inNames = [];
-			const outName = db.makeTempPath("parquet");
-			const registered = [];
-			const bufferedInputs = inputKeys.map((key) => {
-				const name = db.makeTempPath("parquet");
-				inNames.push(name);
-				return {
-					key,
-					name
-				};
-			});
-			try {
-				await registerBufferedFiles(db, bufferedInputs, dataSource, registered, void 0, options.bufferReadConcurrency);
-				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
-				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
-				registered.push(outName);
-				const bytes = await db.copyFileToBuffer(outName);
-				const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
-				const rowCount = Number(countRows[0]?.n ?? 0);
-				await dataSource.write(outputKey, bytes);
-				return {
-					bytes: bytes.byteLength,
-					rowCount
-				};
-			} finally {
-				await db.dropFiles(registered);
-			}
-		}
-	};
-}
-const quoteCol = (c) => `"${c.replace(/"/g, "\"\"")}"`;
-function dedupedMergeSql(table, fileListSql) {
-	const base = `SELECT * FROM read_parquet([${fileListSql}], union_by_name = true)`;
-	const sortKey = SCHEMAS[table].sortKey;
-	const clusterKey = TABLE_METADATA[table].clusterKey;
-	const dedup = sortKey.length === 0 ? base : `${base} QUALIFY row_number() OVER (PARTITION BY ${sortKey.map(quoteCol).join(", ")}) = 1`;
-	if (clusterKey.length === 0) return dedup;
-	return `${dedup} ORDER BY ${clusterKey.map(quoteCol).join(", ")}`;
-}
-function rewriteEmptyFileSets(sql, placeholders, defaultTable, placeholderTables) {
-	let out = sql;
-	for (const [name, keys] of Object.entries(placeholders)) {
-		if (keys.length > 0) continue;
-		const emptyFallback = `(SELECT * FROM ${emptyTableSchema(placeholderTables?.[name] ?? defaultTable)} WHERE FALSE)`;
-		const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${name}\\}\\}\\s*(?:,\\s*union_by_name\\s*=\\s*true\\s*)?\\)`, "g");
-		out = out.replace(pattern, emptyFallback);
-	}
-	return out;
-}
-function createDuckDBExecutor(factory, options = {}) {
-	return { async execute({ sql, params, fileKeys, placeholderTables, dataSource, table, signal, profiler }) {
-		signal?.throwIfAborted();
-		const db = await factory.getDuckDB();
-		const placeholders = {};
-		const registered = [];
-		const totalFiles = Object.values(fileKeys).reduce((n, keys) => n + keys.length, 0);
-		const endRegister = profiler?.start("files.register", { files: totalFiles });
-		try {
-			const bufferedByName = /* @__PURE__ */ new Map();
-			for (const [name, keys] of Object.entries(fileKeys)) {
-				const resolved = [];
-				for (let i = 0; i < keys.length; i++) {
-					const key = keys[i];
-					const uri = dataSource.uri?.(key);
-					if (uri !== void 0) resolved.push(uri);
-					else {
-						if (!bufferedByName.has(key)) bufferedByName.set(key, {
-							key,
-							name: key
-						});
-						resolved.push(key);
-					}
-				}
-				placeholders[name] = resolved;
-			}
-			await registerBufferedFiles(db, [...bufferedByName.values()], dataSource, registered, signal, options.bufferReadConcurrency);
-			endRegister?.({ buffered: registered.length });
-			signal?.throwIfAborted();
-			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
-			const endQuery = profiler?.start("query.run");
-			const rows = await db.query(finalSql, params);
-			endQuery?.({ rows: rows.length });
-			return {
-				rows,
-				sql: finalSql
-			};
-		} finally {
-			if (registered.length > 0) await db.dropFiles(registered);
-		}
-	} };
-}
-function emptyTableSchema(table) {
-	return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
-}
-function dateReplaceClause(table) {
-	if (!table) return "";
-	return dateReplaceClause$1(dateColumnsFor(table), "string");
-}
-function columnList(table) {
-	return SCHEMAS[table].columns.map((c) => c.name).join(", ");
-}
-function placeholderValues(table) {
-	return `(${SCHEMAS[table].columns.map((c) => defaultForType(c.type)).join(", ")})`;
-}
-function defaultForType(t) {
-	if (t === "VARCHAR") return "''";
-	if (t === "DATE") return "DATE '1970-01-01'";
-	if (t === "INTEGER" || t === "BIGINT") return "0";
-	if (t === "DOUBLE") return "CAST(0 AS DOUBLE)";
-	return "NULL";
-}
-function columnsJson(table) {
-	return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
-}
-const DEFAULT_HOURLY_RETENTION_MS = 2160 * 60 * 60 * 1e3;
-const VERSION_RE = /__v(\d+)\.parquet$/;
-function parseLockScope(key) {
-	const match = VERSION_RE.exec(key);
-	if (!match) return void 0;
-	const parts = key.slice(0, match.index).split("/");
-	if (parts.length < 4) return void 0;
-	const userPart = parts[0];
-	if (!userPart.startsWith("u_")) return void 0;
-	const userId = userPart.slice(2);
-	const partition = parts.slice(-2).join("/");
-	const table = parts[parts.length - 3];
-	return {
-		userId,
-		siteId: parts.length >= 5 ? parts.slice(1, -3).join("/") : void 0,
-		table,
-		partition
-	};
-}
-async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
-	const cutoff = now - graceMs;
-	const retired = await deps.manifestStore.listRetired(cutoff);
-	if (retired.length > 0) {
-		await deps.dataSource.delete(retired.map((e) => e.objectKey));
-		await deps.manifestStore.delete(retired);
-	}
-	let hourlyDeleted = 0;
-	if (opts.userId) {
-		const hourlyCutoff = now - (opts.hourlyRetentionMs ?? DEFAULT_HOURLY_RETENTION_MS);
-		const expiredHourly = (await deps.manifestStore.listAll({
-			userId: opts.userId,
-			siteId: opts.siteId
-		})).filter((e) => e.partition.startsWith("hourly/") && e.createdAt < hourlyCutoff);
-		if (expiredHourly.length > 0) {
-			await deps.dataSource.delete(expiredHourly.map((e) => e.objectKey));
-			await deps.manifestStore.delete(expiredHourly);
-			hourlyDeleted = expiredHourly.length;
-		}
-	}
-	let sweptOrphans = 0;
-	if (opts.userId) {
-		const prefix = tenantPrefix({
-			userId: opts.userId,
-			siteId: opts.siteId
-		});
-		const knownEntries = await deps.manifestStore.listAll({
-			userId: opts.userId,
-			siteId: opts.siteId
-		});
-		const knownSet = new Set(knownEntries.map((e) => e.objectKey));
-		const orphans = [];
-		const keyStream = deps.dataSource.streamList ? deps.dataSource.streamList(prefix) : async function* () {
-			const all = await deps.dataSource.list(prefix);
-			for (const k of all) yield k;
-		}();
-		for await (const key of keyStream) {
-			if (knownSet.has(key)) continue;
-			const match = VERSION_RE.exec(key);
-			if (!match) continue;
-			if (Number(match[1]) <= cutoff) orphans.push(key);
-		}
-		const byScope = /* @__PURE__ */ new Map();
-		for (const key of orphans) {
-			const scope = parseLockScope(key);
-			if (!scope) continue;
-			const sk = `${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`;
-			const bucket = byScope.get(sk) ?? {
-				scope,
-				keys: []
-			};
-			bucket.keys.push(key);
-			byScope.set(sk, bucket);
-		}
-		for (const { scope, keys } of byScope.values()) await deps.manifestStore.withLock(scope, async () => {
-			const known = await deps.manifestStore.listAll({
-				userId: scope.userId,
-				siteId: scope.siteId,
-				table: scope.table,
-				partitions: [scope.partition]
-			});
-			const knownInScope = new Set(known.map((e) => e.objectKey));
-			const stillOrphans = keys.filter((k) => !knownInScope.has(k));
-			if (stillOrphans.length > 0) {
-				await deps.dataSource.delete(stillOrphans);
-				sweptOrphans += stillOrphans.length;
-			}
-		});
-	}
-	return { deleted: retired.length + sweptOrphans + hourlyDeleted };
-}
-const PUSHABLE_COLUMN = {
-	country: "country",
-	query: "query",
-	searchAppearance: "searchAppearance"
-};
-function txLeaf(leaf, columns) {
-	if (leaf.operator !== "equals") return null;
-	const column = PUSHABLE_COLUMN[leaf.dimension];
-	if (!column || !columns.has(column)) return null;
-	return { [column]: { $eq: leaf.expression } };
-}
-function combineFilters(parts, groupType) {
-	const first = parts[0];
-	if (!first) return null;
-	if (parts.length === 1) return first;
-	return groupType === "or" ? { $or: [...parts] } : { $and: [...parts] };
-}
-function txExact(node, columns) {
-	const groupType = node._groupType ?? "and";
-	const leafParts = [];
-	for (const leaf of node._filters) {
-		const t = txLeaf(leaf, columns);
-		if (!t) return null;
-		leafParts.push(t);
-	}
-	if (groupType === "or") {
-		if (node._nestedGroups?.length || leafParts.length === 0) return null;
-		return combineFilters(leafParts, "or");
-	}
-	const parts = leafParts;
-	for (const group of node._nestedGroups ?? []) {
-		const t = txExact(group, columns);
-		if (!t) return null;
-		parts.push(t);
-	}
-	if (parts.length === 0) return null;
-	return combineFilters(parts, "and");
-}
-function extractParquetPushdown(state, table) {
-	const filter = state?.filter;
-	const schema = SCHEMAS[table];
-	if (!filter || !schema) return void 0;
-	const columns = new Set(schema.columns.map((c) => c.name));
-	if ((filter._groupType ?? "and") === "or") return txExact(filter, columns) ?? void 0;
-	const parts = [];
-	for (const leaf of filter._filters) {
-		const t = txLeaf(leaf, columns);
-		if (t) parts.push(t);
-	}
-	for (const group of filter._nestedGroups ?? []) {
-		const t = txExact(group, columns);
-		if (t) parts.push(t);
-	}
-	if (parts.length === 0) return void 0;
-	return combineFilters(parts, "and") ?? void 0;
-}
-const URL_PURGE_TABLES = ["pages", "page_queries"];
-const MAX_DAY_BYTES = 100 * 1024 * 1024;
-const URL_COLUMNS = /* @__PURE__ */ new Set();
-for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);
-function normalizeRow(table, row) {
-	if (!URL_COLUMNS.has(`${table}:url`)) return row;
-	const url = row.url;
-	if (typeof url !== "string") return row;
-	const normalized = normalizeUrl(url);
-	if (normalized === url) return row;
-	return {
-		...row,
-		url: normalized
-	};
-}
-const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
-function queryRangeOf(partitions) {
-	if (!partitions) return void 0;
-	let min;
-	let max;
-	for (const p of partitions) {
-		const m = DAILY_PARTITION_RE.exec(p);
-		if (!m) continue;
-		const d = m[1];
-		if (min === void 0 || d < min) min = d;
-		if (max === void 0 || d > max) max = d;
-	}
-	return min !== void 0 ? {
-		start: min,
-		end: max
-	} : void 0;
-}
-function createStorageEngine(opts) {
-	const { dataSource, manifestStore, codec, executor } = opts;
-	const defaultNow = opts.now ?? (() => Date.now());
-	async function writeDay(ctx, rows) {
-		if (!ctx.date) throw new Error("writeDay requires ctx.date");
-		const date = ctx.date;
-		const now = (ctx.now ?? defaultNow)();
-		const partition = dayPartition(date);
-		const searchType = ctx.searchType;
-		return manifestStore.withLock({
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			table: ctx.table,
-			partition
-		}, async () => {
-			const superseding = await manifestStore.listLive({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partitions: [partition],
-				searchType: inferSearchType({ searchType })
-			});
-			const normalizedRows = dedupeByNaturalKey(ctx.table, rows.map((r) => normalizeRow(ctx.table, r)));
-			const key = objectKey(ctx, ctx.table, partition, now, searchType);
-			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
-			let bytes = writtenBytes;
-			if (bytes === 0 && rowCount > 0 && dataSource.head) {
-				const probed = await dataSource.head(key);
-				if (probed) bytes = probed.bytes;
-			}
-			if (bytes > 104857600) {
-				const sizeError = /* @__PURE__ */ new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
-				try {
-					await dataSource.delete([key]);
-				} catch (deleteError) {
-					throw new AggregateError([sizeError, deleteError], `${sizeError.message}; failed to delete oversized object`);
-				}
-				throw sizeError;
-			}
-			const entry = {
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partition,
-				objectKey: key,
-				rowCount,
-				bytes,
-				createdAt: now,
-				schemaVersion: currentSchemaVersion(ctx.table),
-				tier: "raw",
-				...searchType !== void 0 ? { searchType } : {}
-			};
-			await manifestStore.registerVersion(entry, superseding);
-			await manifestStore.bumpWatermark({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				...searchType !== void 0 ? { searchType } : {}
-			}, date, now);
-		});
-	}
-	async function writeHour(ctx, rows) {
-		if (!ctx.date) throw new Error("writeHour requires ctx.date (the PT calendar day)");
-		const date = ctx.date;
-		const now = (ctx.now ?? defaultNow)();
-		const partition = hourPartition(date);
-		const searchType = ctx.searchType;
-		return manifestStore.withLock({
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			table: ctx.table,
-			partition
-		}, async () => {
-			const live = await manifestStore.listLive({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partitions: [partition],
-				searchType: inferSearchType({ searchType })
-			});
-			const existing = [];
-			for (const entry of live) {
-				const rs = await codec.readRows({ table: ctx.table }, entry.objectKey, dataSource);
-				existing.push(...rs);
-			}
-			const dedup = /* @__PURE__ */ new Map();
-			for (const r of existing) {
-				const k = `${String(r.url ?? "")}\0${String(r.hour ?? "")}`;
-				dedup.set(k, r);
-			}
-			for (const r of rows) {
-				const normalized = normalizeRow(ctx.table, r);
-				const k = `${String(normalized.url ?? "")}\0${String(normalized.hour ?? "")}`;
-				dedup.set(k, normalized);
-			}
-			const merged = [...dedup.values()];
-			const key = objectKey(ctx, ctx.table, partition, now, searchType);
-			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, merged, key, dataSource);
-			let bytes = writtenBytes;
-			if (bytes === 0 && rowCount > 0 && dataSource.head) {
-				const probed = await dataSource.head(key);
-				if (probed) bytes = probed.bytes;
-			}
-			if (bytes > 104857600) {
-				const sizeError = /* @__PURE__ */ new Error(`writeHour payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
-				try {
-					await dataSource.delete([key]);
-				} catch (deleteError) {
-					throw new AggregateError([sizeError, deleteError], `${sizeError.message}; failed to delete oversized object`);
-				}
-				throw sizeError;
-			}
-			const entry = {
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partition,
-				objectKey: key,
-				rowCount,
-				bytes,
-				createdAt: now,
-				schemaVersion: currentSchemaVersion(ctx.table),
-				tier: "raw",
-				...searchType !== void 0 ? { searchType } : {}
-			};
-			await manifestStore.registerVersion(entry, live);
-			await manifestStore.bumpWatermark({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				...searchType !== void 0 ? { searchType } : {}
-			}, date, now);
-		});
-	}
-	async function runSQL(opts) {
-		opts.signal?.throwIfAborted();
-		const profiler = opts.profiler;
-		const entries = Object.entries(opts.fileSets);
-		const endList = profiler?.start("manifest.list", { fileSets: entries.length });
-		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
-			if (ref.keys !== void 0) return [name, ref.keys];
-			return [name, dedupeOverlappingTiers(await manifestStore.listLive({
-				userId: opts.ctx.userId,
-				siteId: opts.ctx.siteId,
-				table: ref.table,
-				partitions: ref.partitions,
-				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-			}), queryRangeOf(ref.partitions)).map((e) => e.objectKey)];
-		}));
-		opts.signal?.throwIfAborted();
-		const fileKeys = {};
-		for (const [name, keys] of perSet) fileKeys[name] = keys;
-		const uniqueKeys = [...new Set(perSet.flatMap(([, keys]) => keys))];
-		endList?.({ files: uniqueKeys.length });
-		let table = opts.table;
-		if (!table) {
-			if (new Set(entries.map(([, ref]) => ref.table)).size > 1) throw new Error("runSQL requires explicit ctx.table when fileSets reference multiple tables.");
-			table = entries[0]?.[1].table;
-		}
-		if (!table) throw new Error("runSQL requires at least one fileSet or an explicit table");
-		const placeholderTables = {};
-		for (const [name, ref] of entries) placeholderTables[name] = ref.table;
-		const endExec = profiler?.start("executor.execute", { files: uniqueKeys.length });
-		const result = await executor.execute({
-			sql: opts.sql,
-			params: opts.params ?? [],
-			fileKeys,
-			placeholderTables,
-			dataSource,
-			table,
-			signal: opts.signal,
-			...opts.pushdownFilters ? { pushdownFilters: opts.pushdownFilters } : {},
-			...profiler ? { profiler } : {}
-		});
-		endExec?.({ rows: result.rows.length });
-		return {
-			rows: result.rows,
-			sql: result.sql,
-			objectKeys: uniqueKeys
-		};
-	}
-	async function query(ctx, state) {
-		const plan = buildLogicalPlan(state, { regex: true });
-		const table = ctx.table ?? plan.dataset;
-		const resolved = compileLogicalQueryPlan(plan, table);
-		const pushdown = extractParquetPushdown(state, table);
-		return runSQL({
-			ctx: {
-				userId: ctx.userId,
-				siteId: ctx.siteId
-			},
-			table,
-			fileSets: { FILES: {
-				table,
-				partitions: resolved.partitions
-			} },
-			sql: resolved.sql,
-			params: resolved.params,
-			signal: ctx.signal,
-			...pushdown ? { pushdownFilters: { FILES: pushdown } } : {},
-			...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {},
-			...ctx.profiler ? { profiler: ctx.profiler } : {}
-		});
-	}
-	async function compactTiered(ctx, thresholds) {
-		return compactTieredImpl({
-			dataSource,
-			manifestStore,
-			codec
-		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
-	}
-	async function reconcileSubsumed(ctx) {
-		const { subsumed } = splitOverlappingTiers(await manifestStore.listLive({
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			table: ctx.table
-		}));
-		if (subsumed.length === 0) return {
-			retired: 0,
-			partitions: []
-		};
-		await manifestStore.registerVersions([], subsumed);
-		return {
-			retired: subsumed.length,
-			partitions: subsumed.map((e) => e.partition)
-		};
-	}
-	async function gcOrphans(ctx, graceMs) {
-		return gcOrphansImpl({
-			dataSource,
-			manifestStore
-		}, (ctx.now ?? defaultNow)(), graceMs, {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			...ctx.hourlyRetentionMs !== void 0 ? { hourlyRetentionMs: ctx.hourlyRetentionMs } : {}
-		});
-	}
-	async function purgeTenant(ctx) {
-		const prefix = tenantPrefix(ctx);
-		const keys = [];
-		const keyStream = dataSource.streamList ? dataSource.streamList(prefix) : async function* () {
-			for (const k of await dataSource.list(prefix)) yield k;
-		}();
-		for await (const key of keyStream) keys.push(key);
-		const manifestResult = await manifestStore.purgeTenant({
-			userId: ctx.userId,
-			siteId: ctx.siteId
-		});
-		if (keys.length > 0) await dataSource.delete(keys);
-		return {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			prefix,
-			objectsDeleted: keys.length,
-			entriesRemoved: manifestResult.entriesRemoved,
-			watermarksRemoved: manifestResult.watermarksRemoved,
-			syncStatesRemoved: manifestResult.syncStatesRemoved,
-			at: defaultNow()
-		};
-	}
-	async function purgeUrls(ctx, urls) {
-		const now = defaultNow();
-		const urlSet = new Set(urls);
-		let entriesRewritten = 0;
-		let rowsRemoved = 0;
-		let bytesAfter = 0;
-		if (urlSet.size === 0) return {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			urlsRequested: 0,
-			entriesRewritten: 0,
-			rowsRemoved: 0,
-			bytesAfter: 0,
-			at: now
-		};
-		for (const table of URL_PURGE_TABLES) {
-			const entries = await manifestStore.listLive({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table
-			});
-			for (const entry of entries) await manifestStore.withLock({
-				userId: entry.userId,
-				siteId: entry.siteId,
-				table,
-				partition: entry.partition
-			}, async () => {
-				const rows = await codec.readRows({ table }, entry.objectKey, dataSource);
-				const kept = rows.filter((r) => typeof r.url !== "string" || !urlSet.has(r.url));
-				const removed = rows.length - kept.length;
-				if (removed === 0) return;
-				const searchType = entry.searchType;
-				const newKey = objectKey({
-					userId: entry.userId,
-					siteId: entry.siteId
-				}, table, entry.partition, now, searchType);
-				const { bytes, rowCount } = await codec.writeRows({ table }, kept, newKey, dataSource);
-				const newEntry = {
-					userId: entry.userId,
-					siteId: entry.siteId,
-					table,
-					partition: entry.partition,
-					objectKey: newKey,
-					rowCount,
-					bytes,
-					createdAt: now,
-					schemaVersion: entry.schemaVersion ?? currentSchemaVersion(table),
-					...entry.tier !== void 0 ? { tier: entry.tier } : {},
-					...searchType !== void 0 ? { searchType } : {}
-				};
-				await manifestStore.registerVersion(newEntry, [entry]);
-				entriesRewritten++;
-				rowsRemoved += removed;
-				bytesAfter += bytes;
-			});
-		}
-		return {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			urlsRequested: urlSet.size,
-			entriesRewritten,
-			rowsRemoved,
-			bytesAfter,
-			at: now
-		};
-	}
-	return {
-		writeDay,
-		writeHour,
-		query,
-		runSQL,
-		compactTiered,
-		reconcileSubsumed,
-		gcOrphans,
-		purgeTenant,
-		purgeUrls,
-		listLive: (filter) => manifestStore.listLive(filter),
-		listAll: (filter) => manifestStore.listAll(filter),
-		getWatermarks: (filter) => manifestStore.getWatermarks(filter),
-		getSyncStates: (filter) => manifestStore.getSyncStates(filter),
-		setSyncState: (scope, state, detail) => manifestStore.setSyncState(scope, state, detail),
-		readObject: (key) => dataSource.read(key)
-	};
-}
-export { MAX_DAY_BYTES, createDuckDBCodec, createDuckDBExecutor, createStorageEngine };