@gscdump/engine 0.4.0

package/dist/adapters/node-harness.mjs

@@ -0,0 +1,1907 @@
+import { createRequire } from "node:module";
+import path, { dirname, join, resolve } from "node:path";
+import { encodeSiteId } from "gscdump/tenant";
+import { buildLogicalComparisonPlan, buildLogicalPlan } from "gscdump/query/plan";
+import { MS_PER_DAY, toIsoDate } from "gscdump";
+import { PgDialect, date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
+import { normalizeUrl } from "gscdump/normalize";
+import { sql } from "drizzle-orm";
+import { SQLiteAsyncDialect } from "drizzle-orm/sqlite-core";
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
+import { Buffer } from "node:buffer";
+import { randomBytes } from "node:crypto";
+import { mkdir, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { lock } from "proper-lockfile";
+function metricCols() {
+	return {
+		clicks: integer("clicks").notNull(),
+		impressions: integer("impressions").notNull(),
+		sum_position: doublePrecision("sum_position").notNull()
+	};
+}
+const dateCol = () => date("date").notNull();
+const drizzleSchema = {
+	pages: pgTable("pages", {
+		url: varchar("url").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	keywords: pgTable("keywords", {
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	countries: pgTable("countries", {
+		country: varchar("country").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	devices: pgTable("devices", {
+		device: varchar("device").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	page_keywords: pgTable("page_keywords", {
+		url: varchar("url").notNull(),
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	search_appearance: pgTable("search_appearance", {
+		searchAppearance: varchar("searchAppearance").notNull(),
+		date: dateCol(),
+		...metricCols()
+	})
+};
+const TABLE_METADATA = {
+	pages: {
+		sortKey: ["date", "url"],
+		version: 1
+	},
+	keywords: {
+		sortKey: ["date", "query"],
+		version: 2
+	},
+	countries: {
+		sortKey: ["date", "country"],
+		version: 1
+	},
+	devices: {
+		sortKey: ["date", "device"],
+		version: 1
+	},
+	page_keywords: {
+		sortKey: [
+			"date",
+			"url",
+			"query"
+		],
+		version: 2
+	},
+	search_appearance: {
+		sortKey: ["date", "searchAppearance"],
+		version: 1
+	}
+};
+function pgSqlTypeToColumnType(sqlType) {
+	const t = sqlType.toLowerCase();
+	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
+	if (t === "date" || t.startsWith("timestamp")) return "DATE";
+	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
+	if (t === "bigint" || t === "int8") return "BIGINT";
+	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
+	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
+}
+function tableSchemaFrom(tableName) {
+	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
+		name: col.name,
+		type: pgSqlTypeToColumnType(col.getSQLType()),
+		nullable: !col.notNull
+	}));
+	const meta = TABLE_METADATA[tableName];
+	return {
+		name: tableName,
+		columns,
+		sortKey: meta.sortKey,
+		version: meta.version
+	};
+}
+const SCHEMAS = Object.fromEntries([
+	"pages",
+	"keywords",
+	"countries",
+	"devices",
+	"page_keywords",
+	"search_appearance"
+].map((t) => [t, tableSchemaFrom(t)]));
+function currentSchemaVersion(table) {
+	return SCHEMAS[table].version;
+}
+function dimensionToColumn(dim, _table) {
+	if (dim === "page") return "url";
+	if (dim === "queryCanonical") return "query_canonical";
+	return dim;
+}
+function inferSearchType(entry) {
+	return entry.searchType ?? "web";
+}
+function inferLegacyTier(entry) {
+	if (entry.tier !== void 0) return entry.tier;
+	if (entry.partition.startsWith("daily/")) return "raw";
+	if (entry.partition.startsWith("monthly/")) return "d30";
+}
+function dayPartition(date) {
+	return `daily/${date}`;
+}
+function monthPartition(month) {
+	return `monthly/${month}`;
+}
+function weekPartition(mondayIsoDate) {
+	return `weekly/${mondayIsoDate}`;
+}
+function quarterPartition(quarter) {
+	return `quarterly/${quarter}`;
+}
+function mondayOfWeek(isoDate) {
+	const ms = Date.parse(`${isoDate}T00:00:00Z`);
+	const dow = new Date(ms).getUTCDay();
+	const offset = dow === 0 ? -6 : 1 - dow;
+	return toIsoDate(new Date(ms + offset * MS_PER_DAY));
+}
+function quarterOfMonth(month) {
+	const [y, m] = month.split("-").map(Number);
+	return `${y}-Q${Math.floor((m - 1) / 3) + 1}`;
+}
+function objectKey(ctx, table, partition, version, searchType) {
+	return `${ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/${table}` : `u_${ctx.userId}/${table}`}/${searchType !== void 0 && searchType !== "web" ? `${searchType}/` : ""}${partition}__v${version}.parquet`;
+}
+function tenantPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/` : `u_${ctx.userId}/`;
+}
+const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
+const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
+const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
+const DEFAULT_THRESHOLDS = {
+	raw: 7,
+	d7: 30,
+	d30: 90
+};
+const PENDING_WINDOW_DAYS = 4;
+const STAGES = [
+	{
+		inputTier: "raw",
+		outputTier: "d7",
+		cutoffDays: DEFAULT_THRESHOLDS.raw,
+		bucketKey: (e) => {
+			const m = e.partition.match(DAILY_PARTITION_RE);
+			if (!m) return void 0;
+			return mondayOfWeek(m[1]);
+		},
+		bucketLatestMs: (monday) => Date.parse(`${monday}T00:00:00Z`) + 6 * MS_PER_DAY,
+		outputPartition: weekPartition
+	},
+	{
+		inputTier: "d7",
+		outputTier: "d30",
+		cutoffDays: DEFAULT_THRESHOLDS.d7,
+		bucketKey: (e) => {
+			const m = e.partition.match(WEEKLY_PARTITION_RE);
+			if (!m) return void 0;
+			return m[1].slice(0, 7);
+		},
+		bucketLatestMs: monthEndMs,
+		outputPartition: monthPartition
+	},
+	{
+		inputTier: "d30",
+		outputTier: "d90",
+		cutoffDays: DEFAULT_THRESHOLDS.d30,
+		bucketKey: (e) => {
+			const m = e.partition.match(MONTHLY_PARTITION_RE);
+			if (!m) return void 0;
+			return quarterOfMonth(m[1]);
+		},
+		bucketLatestMs: quarterEndMs,
+		outputPartition: quarterPartition
+	}
+];
+async function compactTieredImpl(deps, ctx, now, overrides = {}) {
+	const thresholds = {
+		...DEFAULT_THRESHOLDS,
+		...overrides
+	};
+	const stagesWithThresholds = STAGES.map((s) => ({
+		...s,
+		cutoffDays: s.outputTier === "d7" ? thresholds.raw : s.outputTier === "d30" ? thresholds.d7 : thresholds.d30
+	}));
+	for (const stage of stagesWithThresholds) await runStage(deps, ctx, stage, now);
+}
+async function runStage(deps, ctx, stage, now) {
+	const cutoff = now - Math.max(stage.cutoffDays, PENDING_WINDOW_DAYS) * MS_PER_DAY;
+	const candidates = await deps.manifestStore.listLive({
+		userId: ctx.userId,
+		siteId: ctx.siteId,
+		table: ctx.table,
+		tier: stage.inputTier
+	});
+	const buckets = /* @__PURE__ */ new Map();
+	for (const entry of candidates) {
+		const key = stage.bucketKey(entry);
+		if (!key) continue;
+		if (stage.bucketLatestMs(key) >= cutoff) continue;
+		const compositeKey = `${inferSearchType(entry)}\0${key}`;
+		if (!buckets.has(compositeKey)) buckets.set(compositeKey, []);
+		buckets.get(compositeKey).push(entry);
+	}
+	for (const [compositeKey, entries] of buckets) {
+		const [searchType, bucket] = compositeKey.split("\0");
+		const targetPartition = stage.outputPartition(bucket);
+		if (entries.length === 1 && entries[0].partition === targetPartition) continue;
+		await deps.manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition: targetPartition
+		}, async () => {
+			const key = objectKey(ctx, ctx.table, targetPartition, now, searchType);
+			const { bytes, rowCount } = await deps.codec.compactRows({ table: ctx.table }, entries.map((e) => e.objectKey), key, deps.dataSource);
+			const newEntry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition: targetPartition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: stage.outputTier,
+				...searchType !== "web" ? { searchType } : {}
+			};
+			await deps.manifestStore.registerVersion(newEntry, entries);
+		});
+	}
+}
+function enumeratePartitions(startDate, endDate) {
+	const out = [];
+	const [sy, sm, sd] = startDate.split("-").map(Number);
+	const [ey, em, ed] = endDate.split("-").map(Number);
+	const start = Date.UTC(sy, sm - 1, sd);
+	const end = Date.UTC(ey, em - 1, ed);
+	if (end < start) return out;
+	const seenWeeks = /* @__PURE__ */ new Set();
+	const seenMonths = /* @__PURE__ */ new Set();
+	const seenQuarters = /* @__PURE__ */ new Set();
+	for (let t = start; t <= end; t += 864e5) {
+		const d = new Date(t);
+		const y = d.getUTCFullYear();
+		const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+		const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
+		const isoMonth = `${y}-${m}`;
+		out.push(dayPartition(isoDay));
+		const monday = mondayOfWeek(isoDay);
+		if (!seenWeeks.has(monday)) {
+			seenWeeks.add(monday);
+			out.push(weekPartition(monday));
+		}
+		if (!seenMonths.has(isoMonth)) {
+			seenMonths.add(isoMonth);
+			out.push(monthPartition(isoMonth));
+		}
+		const quarter = quarterOfMonth(isoMonth);
+		if (!seenQuarters.has(quarter)) {
+			seenQuarters.add(quarter);
+			out.push(quarterPartition(quarter));
+		}
+	}
+	return out;
+}
+function monthEndMs(month) {
+	const [y, m] = month.split("-").map(Number);
+	return Date.UTC(y, m, 0, 23, 59, 59, 999);
+}
+function quarterEndMs(quarter) {
+	const [yStr, qStr] = quarter.split("-Q");
+	const y = Number(yStr);
+	const q = Number(qStr);
+	return Date.UTC(y, q * 3, 0, 23, 59, 59, 999);
+}
+function escapeLike(value) {
+	return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
+}
+const METRIC_EXPR = {
+	clicks: "CAST(SUM(clicks) AS DOUBLE)",
+	impressions: "CAST(SUM(impressions) AS DOUBLE)",
+	ctr: "CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0)",
+	position: "SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1"
+};
+function topLevelPagePredicateSql(pathExpr) {
+	return `LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
+}
+const FILES_PLACEHOLDER = "{{FILES}}";
+function buildDimensionWhere(filters, table) {
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const column = dimensionToColumn(filter.dimension, table);
+		switch (filter.operator) {
+			case "equals":
+				clauses.push(`${column} = ?`);
+				params.push(filter.expression);
+				break;
+			case "notEquals":
+				clauses.push(`${column} != ?`);
+				params.push(filter.expression);
+				break;
+			case "contains":
+				clauses.push(`${column} LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "notContains":
+				clauses.push(`${column} NOT LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "includingRegex":
+				clauses.push(`regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+			case "excludingRegex":
+				clauses.push(`NOT regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.join(" AND "),
+		params
+	};
+}
+function buildTopLevelWhere(plan, table) {
+	if (!plan.specialFilters.topLevel) return "";
+	return topLevelPagePredicateSql(dimensionToColumn("page", table));
+}
+function buildHaving(filters) {
+	if (filters.length === 0) return {
+		clause: "",
+		params: []
+	};
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const expr = METRIC_EXPR[filter.metric];
+		switch (filter.operator) {
+			case "metricGte":
+				clauses.push(`${expr} >= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricGt":
+				clauses.push(`${expr} > ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLte":
+				clauses.push(`${expr} <= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLt":
+				clauses.push(`${expr} < ?`);
+				params.push(filter.expression);
+				break;
+			case "metricBetween":
+				clauses.push(`${expr} >= ? AND ${expr} <= ?`);
+				params.push(filter.expression, filter.expression2 ?? filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.length > 0 ? `HAVING ${clauses.join(" AND ")}` : "",
+		params
+	};
+}
+function compileLogicalQueryPlan(plan, table = plan.dataset) {
+	const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
+	const metricSelects = plan.metrics.map((metric) => `${METRIC_EXPR[metric]} AS ${metric}`);
+	const dimSelects = plan.groupByDimensions.map((dimension) => {
+		const column = dimensionToColumn(dimension, table);
+		return column !== dimension ? `${column} AS ${dimension}` : dimension;
+	});
+	const whereClauses = ["date >= ?", "date <= ?"];
+	const whereParams = [plan.dateRange.startDate, plan.dateRange.endDate];
+	const dimWhere = buildDimensionWhere(plan.dimensionFilters, table);
+	if (dimWhere.clause) {
+		whereClauses.push(dimWhere.clause);
+		whereParams.push(...dimWhere.params);
+	}
+	const topLevelClause = buildTopLevelWhere(plan, table);
+	if (topLevelClause) whereClauses.push(topLevelClause);
+	const having = buildHaving(plan.metricFilters);
+	const groupByCols = [...plan.groupByDimensions.map((dimension) => dimensionToColumn(dimension, table)), ...plan.hasDate ? ["date"] : []];
+	const groupBy = groupByCols.length > 0 ? `GROUP BY ${groupByCols.join(", ")}` : "";
+	const orderBy = plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC";
+	const limit = `LIMIT ${plan.rowLimit ?? 1e3}`;
+	const offset = plan.startRow ? `OFFSET ${plan.startRow}` : "";
+	return {
+		sql: [
+			`SELECT ${[
+				...dimSelects,
+				...plan.hasDate ? ["date"] : [],
+				...metricSelects
+			].join(", ")}`,
+			`FROM read_parquet(${FILES_PLACEHOLDER}, union_by_name = true)`,
+			`WHERE ${whereClauses.join(" AND ")}`,
+			groupBy,
+			having.clause,
+			orderBy,
+			limit,
+			offset
+		].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(),
+		params: [...whereParams, ...having.params],
+		partitions,
+		table,
+		filesPlaceholder: FILES_PLACEHOLDER
+	};
+}
+function fileList(keys) {
+	return keys.length === 0 ? "[]" : `[${keys.map((key) => `'${key.replace(/'/g, "''")}'`).join(", ")}]`;
+}
+function substituteNamedFiles(sql, sets) {
+	let out = sql;
+	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
+	return out;
+}
+function sqlEscape(s) {
+	return s.replace(/'/g, "''");
+}
+async function encodeBytes(db, table, rows) {
+	const inName = db.makeTempPath("json");
+	const outName = db.makeTempPath("parquet");
+	const jsonBytes = new TextEncoder().encode(JSON.stringify(rows));
+	const registered = [];
+	await db.registerFileBuffer(inName, jsonBytes);
+	registered.push(inName);
+	try {
+		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`;
+		await db.query(sql);
+		registered.push(outName);
+		return await db.copyFileToBuffer(outName);
+	} finally {
+		await db.dropFiles(registered);
+	}
+}
+async function decodeBytes(db, bytes, table) {
+	const name = db.makeTempPath("parquet");
+	await db.registerFileBuffer(name, bytes);
+	try {
+		return await db.query(`SELECT * ${dateReplaceClause(table)} FROM read_parquet('${sqlEscape(name)}')`);
+	} finally {
+		await db.dropFiles([name]);
+	}
+}
+function createDuckDBCodec(factory) {
+	return {
+		async writeRows(ctx, rows, key, dataSource) {
+			const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
+			await dataSource.write(key, bytes);
+			return {
+				bytes: bytes.byteLength,
+				rowCount: rows.length
+			};
+		},
+		async readRows(ctx, key, dataSource) {
+			return decodeBytes(await factory.getDuckDB(), await dataSource.read(key), ctx.table);
+		},
+		async compactRows(ctx, inputKeys, outputKey, dataSource) {
+			const db = await factory.getDuckDB();
+			if (inputKeys.length === 0) {
+				const bytes = await encodeBytes(db, ctx.table, []);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount: 0
+				};
+			}
+			const inputUris = inputKeys.map((k) => dataSource.uri?.(k));
+			if (inputUris.every((u) => u !== void 0)) {
+				const outName = db.makeTempPath("parquet");
+				const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
+				try {
+					await db.query(`COPY (SELECT * FROM read_parquet([${fileList}], union_by_name=true)) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
+					const bytes = await db.copyFileToBuffer(outName);
+					const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
+					const rowCount = Number(countRows[0]?.n ?? 0);
+					await dataSource.write(outputKey, bytes);
+					return {
+						bytes: bytes.byteLength,
+						rowCount
+					};
+				} finally {
+					await db.dropFiles([outName]);
+				}
+			}
+			const inputs = await Promise.all(inputKeys.map((k) => dataSource.read(k)));
+			const inNames = [];
+			const outName = db.makeTempPath("parquet");
+			const registered = [];
+			for (let i = 0; i < inputs.length; i++) {
+				const name = db.makeTempPath("parquet");
+				await db.registerFileBuffer(name, inputs[i]);
+				inNames.push(name);
+				registered.push(name);
+			}
+			try {
+				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
+				await db.query(`COPY (SELECT * FROM read_parquet([${fileList}], union_by_name = true)) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
+				registered.push(outName);
+				const bytes = await db.copyFileToBuffer(outName);
+				const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
+				const rowCount = Number(countRows[0]?.n ?? 0);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount
+				};
+			} finally {
+				await db.dropFiles(registered);
+			}
+		}
+	};
+}
+function rewriteEmptyFileSets(sql, placeholders, table) {
+	const emptyFallback = `(SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE)`;
+	let out = sql;
+	for (const [name, keys] of Object.entries(placeholders)) {
+		if (keys.length > 0) continue;
+		const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${name}\\}\\}\\s*(?:,\\s*union_by_name\\s*=\\s*true\\s*)?\\)`, "g");
+		out = out.replace(pattern, emptyFallback);
+	}
+	return out;
+}
+function createDuckDBExecutor(factory) {
+	return { async execute({ sql, params, fileKeys, dataSource, table, signal }) {
+		signal?.throwIfAborted();
+		const db = await factory.getDuckDB();
+		const placeholders = {};
+		const registered = [];
+		for (const [name, keys] of Object.entries(fileKeys)) {
+			const resolved = [];
+			for (const key of keys) {
+				const uri = dataSource.uri?.(key);
+				if (uri !== void 0) resolved.push(uri);
+				else {
+					const bytes = await dataSource.read(key, void 0, signal);
+					await db.registerFileBuffer(key, bytes);
+					registered.push(key);
+					resolved.push(key);
+				}
+			}
+			placeholders[name] = resolved;
+		}
+		try {
+			signal?.throwIfAborted();
+			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table), placeholders);
+			return {
+				rows: await db.query(finalSql, params),
+				sql: finalSql
+			};
+		} finally {
+			if (registered.length > 0) await db.dropFiles(registered);
+		}
+	} };
+}
+function emptyTableSchema(table) {
+	return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
+}
+function dateReplaceClause(table) {
+	if (!table) return "";
+	const dateCols = SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
+	if (dateCols.length === 0) return "";
+	return `REPLACE (${dateCols.map((n) => `strftime(${n}, '%Y-%m-%d') AS ${n}`).join(", ")})`;
+}
+function columnList(table) {
+	return SCHEMAS[table].columns.map((c) => c.name).join(", ");
+}
+function placeholderValues(table) {
+	return `(${SCHEMAS[table].columns.map((c) => defaultForType(c.type)).join(", ")})`;
+}
+function defaultForType(t) {
+	if (t === "VARCHAR") return "''";
+	if (t === "DATE") return "DATE '1970-01-01'";
+	if (t === "INTEGER" || t === "BIGINT") return "0";
+	if (t === "DOUBLE") return "CAST(0 AS DOUBLE)";
+	return "NULL";
+}
+function columnsJson(table) {
+	return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
+}
+const VERSION_RE = /__v(\d+)\.parquet$/;
+function parseLockScope(key) {
+	const match = VERSION_RE.exec(key);
+	if (!match) return void 0;
+	const parts = key.slice(0, match.index).split("/");
+	if (parts.length < 4) return void 0;
+	const userPart = parts[0];
+	if (!userPart.startsWith("u_")) return void 0;
+	const userId = userPart.slice(2);
+	const partition = parts.slice(-2).join("/");
+	const table = parts[parts.length - 3];
+	return {
+		userId,
+		siteId: parts.length >= 5 ? parts.slice(1, -3).join("/") : void 0,
+		table,
+		partition
+	};
+}
+async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
+	const cutoff = now - graceMs;
+	const retired = await deps.manifestStore.listRetired(cutoff);
+	if (retired.length > 0) {
+		await deps.dataSource.delete(retired.map((e) => e.objectKey));
+		await deps.manifestStore.delete(retired);
+	}
+	let sweptOrphans = 0;
+	if (opts.userId) {
+		const prefix = tenantPrefix({
+			userId: opts.userId,
+			siteId: opts.siteId
+		});
+		const knownEntries = await deps.manifestStore.listAll({
+			userId: opts.userId,
+			siteId: opts.siteId
+		});
+		const knownSet = new Set(knownEntries.map((e) => e.objectKey));
+		const orphans = [];
+		const keyStream = deps.dataSource.streamList ? deps.dataSource.streamList(prefix) : async function* () {
+			const all = await deps.dataSource.list(prefix);
+			for (const k of all) yield k;
+		}();
+		for await (const key of keyStream) {
+			if (knownSet.has(key)) continue;
+			const match = VERSION_RE.exec(key);
+			if (!match) continue;
+			if (Number(match[1]) <= cutoff) orphans.push(key);
+		}
+		const byScope = /* @__PURE__ */ new Map();
+		for (const key of orphans) {
+			const scope = parseLockScope(key);
+			if (!scope) continue;
+			const sk = `${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`;
+			const bucket = byScope.get(sk) ?? {
+				scope,
+				keys: []
+			};
+			bucket.keys.push(key);
+			byScope.set(sk, bucket);
+		}
+		for (const { scope, keys } of byScope.values()) await deps.manifestStore.withLock(scope, async () => {
+			const known = await deps.manifestStore.listAll({
+				userId: scope.userId,
+				siteId: scope.siteId,
+				table: scope.table,
+				partitions: [scope.partition]
+			});
+			const knownInScope = new Set(known.map((e) => e.objectKey));
+			const stillOrphans = keys.filter((k) => !knownInScope.has(k));
+			if (stillOrphans.length > 0) {
+				await deps.dataSource.delete(stillOrphans);
+				sweptOrphans += stillOrphans.length;
+			}
+		});
+	}
+	return { deleted: retired.length + sweptOrphans };
+}
+const COMPARISON_FILTER_SQL = {
+	new: sql`AND (p.impressions IS NULL OR p.impressions = 0)`,
+	lost: sql`AND p.impressions > 0 AND c.impressions = 0`,
+	improving: sql`AND c.clicks > COALESCE(p.clicks, 0)`,
+	declining: sql`AND c.clicks < p.clicks AND p.clicks > 0`
+};
+function collapseWs(s) {
+	return s.replace(/\s+/g, " ").trim();
+}
+function joinAnd(parts) {
+	return sql.join(parts, sql` AND `);
+}
+function joinComma(parts) {
+	return sql.join(parts, sql`, `);
+}
+function orderByClause(state, prefix = "") {
+	if (state.orderBy) {
+		const safeCol = state.orderBy.column.replace(/\W/g, "");
+		const safeDir = state.orderBy.dir.toUpperCase() === "ASC" ? "ASC" : "DESC";
+		return sql.raw(`ORDER BY ${prefix}${safeCol} ${safeDir}`);
+	}
+	return sql.raw(`ORDER BY ${prefix}clicks DESC`);
+}
+function limitOffsetClause(state) {
+	const rowLimit = Math.max(0, Math.floor(Number(state.rowLimit ?? 100)));
+	const offset = state.startRow ? Math.max(0, Math.floor(Number(state.startRow))) : 0;
+	return sql.raw(offset > 0 ? `LIMIT ${rowLimit} OFFSET ${offset}` : `LIMIT ${rowLimit}`);
+}
+function aliasRaw(name) {
+	const safe = name.replace(/\W/g, "");
+	return sql.raw(`"${safe}"`);
+}
+function toInternalDimensionFilters(filters) {
+	return filters.map((filter) => ({
+		dimension: filter.dimension,
+		operator: filter.operator,
+		expression: filter.expression,
+		expression2: filter.expression2
+	}));
+}
+function toInternalMetricFilters(filters) {
+	return filters.map((filter) => ({
+		dimension: filter.metric,
+		operator: filter.operator,
+		expression: String(filter.expression),
+		expression2: filter.expression2 == null ? void 0 : String(filter.expression2)
+	}));
+}
+function topLevelFilters(plan) {
+	if (!plan.specialFilters.topLevel) return [];
+	return [{
+		dimension: "page",
+		operator: "topLevel",
+		expression: ""
+	}];
+}
+function logicalFilterToInternal(filter) {
+	return {
+		dimension: filter.dimension,
+		operator: filter.operator,
+		expression: filter.expression,
+		expression2: filter.expression2
+	};
+}
+function compileFilterTree(node, adapter, tableKey) {
+	if (!node) return void 0;
+	if (node.kind === "leaf") return adapter.dimensionPredicates([logicalFilterToInternal(node.filter)], tableKey)[0];
+	const childSqls = node.children.map((child) => compileFilterTree(child, adapter, tableKey)).filter((s) => s !== void 0);
+	if (childSqls.length === 0) return void 0;
+	if (childSqls.length === 1) return childSqls[0];
+	const sep = node.groupType === "or" ? sql` OR ` : sql` AND `;
+	return sql`(${sql.join(childSqls, sep)})`;
+}
+function buildScope(state, options) {
+	const { adapter, siteId } = options;
+	const plan = buildLogicalPlan(state, adapter.capabilities);
+	const tableKey = adapter.tableKeyForDataset(plan.dataset);
+	const dimFilters = toInternalDimensionFilters(plan.dimensionFilters);
+	const metricFilters = toInternalMetricFilters(plan.metricFilters);
+	const groupByDims = plan.groupByDimensions;
+	const hasDate = plan.hasDate;
+	const metrics = plan.metrics;
+	const wherePredicates = [];
+	if (adapter.siteIdColRef && siteId != null) wherePredicates.push(sql`${adapter.siteIdColRef(tableKey)} = ${siteId}`);
+	wherePredicates.push(sql`${adapter.dateColRef(tableKey)} >= ${plan.dateRange.startDate}`);
+	wherePredicates.push(sql`${adapter.dateColRef(tableKey)} <= ${plan.dateRange.endDate}`);
+	const dimSql = plan.dimensionFilterTree ? compileFilterTree(plan.dimensionFilterTree, adapter, tableKey) : void 0;
+	if (dimSql) wherePredicates.push(dimSql);
+	else if (!plan.dimensionFilterTree) wherePredicates.push(...adapter.dimensionPredicates(dimFilters, tableKey));
+	const tl = adapter.topLevelPredicate(topLevelFilters(plan), tableKey);
+	if (tl) wherePredicates.push(tl);
+	return {
+		plan,
+		tableKey,
+		groupByDims,
+		hasDate,
+		metrics,
+		wherePredicates,
+		having: adapter.havingPredicates(metricFilters, tableKey),
+		dimFilters,
+		startDate: plan.dateRange.startDate,
+		endDate: plan.dateRange.endDate
+	};
+}
+function buildComparisonPlan(current, previous, capabilities) {
+	return buildLogicalComparisonPlan(current, previous, capabilities);
+}
+function compileCollapsed(adapter, q) {
+	const c = adapter.compile(q);
+	return {
+		sql: collapseWs(c.sql),
+		params: c.params
+	};
+}
+function buildTotalsSql(state, options) {
+	const { adapter } = options;
+	const { tableKey, metrics, wherePredicates } = buildScope(state, options);
+	const table = adapter.tableRef(tableKey);
+	const selectExprs = metrics.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`);
+	return compileCollapsed(adapter, wherePredicates.length > 0 ? sql`SELECT ${joinComma(selectExprs)} FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT ${joinComma(selectExprs)} FROM ${table}`);
+}
+function resolveComparisonSQL(current, previous, options, comparisonFilter) {
+	const { adapter, siteId } = options;
+	const comparisonPlan = buildComparisonPlan(current, previous, adapter.capabilities);
+	const currentScope = buildScope(current, options);
+	const previousScope = buildScope(previous, options);
+	const { tableKey, groupByDims, metrics, wherePredicates: currentWhere, having } = currentScope;
+	const table = adapter.tableRef(tableKey);
+	const dimSelectExprs = [];
+	for (const d of groupByDims) {
+		const expr = adapter.dimExprSql(d, tableKey);
+		const colName = adapter.dimColumn(d, tableKey);
+		if (d === "page" || colName !== d) dimSelectExprs.push(sql`${expr} as ${aliasRaw(d)}`);
+		else dimSelectExprs.push(expr);
+	}
+	const currentSelect = [...dimSelectExprs, ...metrics.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`)];
+	const prevSelect = [...dimSelectExprs, ...adapter.METRIC_NAMES.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`)];
+	const groupByExprs = groupByDims.map((d) => adapter.dimExprSql(d, tableKey));
+	const prevWhere = [];
+	if (adapter.siteIdColRef && siteId != null) prevWhere.push(sql`${adapter.siteIdColRef(tableKey)} = ${siteId}`);
+	if (previousScope.startDate) prevWhere.push(sql`${adapter.dateColRef(tableKey)} >= ${previousScope.startDate}`);
+	if (previousScope.endDate) prevWhere.push(sql`${adapter.dateColRef(tableKey)} <= ${previousScope.endDate}`);
+	const prevDimSql = comparisonPlan.current.dimensionFilterTree ? compileFilterTree(comparisonPlan.current.dimensionFilterTree, adapter, tableKey) : void 0;
+	if (prevDimSql) prevWhere.push(prevDimSql);
+	else if (!comparisonPlan.current.dimensionFilterTree) prevWhere.push(...adapter.dimensionPredicates(toInternalDimensionFilters(comparisonPlan.current.dimensionFilters), tableKey));
+	let currentCte = currentWhere.length > 0 ? sql`SELECT ${joinComma(currentSelect)} FROM ${table} WHERE ${joinAnd(currentWhere)}` : sql`SELECT ${joinComma(currentSelect)} FROM ${table}`;
+	if (groupByExprs.length > 0) currentCte = sql`${currentCte} GROUP BY ${joinComma(groupByExprs)}`;
+	if (having.length > 0) currentCte = sql`${currentCte} HAVING ${joinAnd(having)}`;
+	let previousCte = prevWhere.length > 0 ? sql`SELECT ${joinComma(prevSelect)} FROM ${table} WHERE ${joinAnd(prevWhere)}` : sql`SELECT ${joinComma(prevSelect)} FROM ${table}`;
+	if (groupByExprs.length > 0) previousCte = sql`${previousCte} GROUP BY ${joinComma(groupByExprs)}`;
+	const joinOn = groupByDims.length > 0 ? sql.raw(groupByDims.map((d) => `c.${d.replace(/\W/g, "")} = p.${d.replace(/\W/g, "")}`).join(" AND ")) : sql.raw("1=1");
+	const filterClause = comparisonFilter ? COMPARISON_FILTER_SQL[comparisonFilter] : sql.raw("");
+	const orderSql = orderByClause(current, "c.");
+	const limitSql = limitOffsetClause(current);
+	const outerCurrentCols = [];
+	for (const d of groupByDims) {
+		const colName = d.replace(/\W/g, "");
+		outerCurrentCols.push(sql.raw(`c.${colName} as "${colName}"`));
+	}
+	outerCurrentCols.push(sql.raw("CAST(c.clicks AS DOUBLE) as \"clicks\""));
+	outerCurrentCols.push(sql.raw("CAST(c.impressions AS DOUBLE) as \"impressions\""));
+	outerCurrentCols.push(sql.raw("c.ctr as \"ctr\""));
+	outerCurrentCols.push(sql.raw("c.position as \"position\""));
+	const mainQuery = sql`WITH current AS (${currentCte}), previous AS (${previousCte}) SELECT ${joinComma(outerCurrentCols)}, COALESCE(CAST(p.clicks AS DOUBLE), 0) as "prevClicks", COALESCE(CAST(p.impressions AS DOUBLE), 0) as "prevImpressions", COALESCE(p.ctr, 0) as "prevCtr", COALESCE(p.position, 0) as "prevPosition" FROM current c LEFT JOIN previous p ON ${joinOn} WHERE 1=1 ${filterClause} ${orderSql} ${limitSql}`;
+	const firstGroupBy = groupByDims[0] ? groupByDims[0].replace(/\W/g, "") : "clicks";
+	const countInnerSelect = sql.raw(`c.${firstGroupBy}`);
+	const countQuery = sql`WITH current AS (${currentCte}), previous AS (${previousCte}) SELECT COUNT(*) as total FROM (SELECT ${countInnerSelect} FROM current c LEFT JOIN previous p ON ${joinOn} WHERE 1=1 ${filterClause})`;
+	const main = compileCollapsed(adapter, mainQuery);
+	const count = compileCollapsed(adapter, countQuery);
+	return {
+		sql: main.sql,
+		params: main.params,
+		countSql: count.sql,
+		countParams: count.params
+	};
+}
+function buildExtrasQueries(state, options) {
+	const { adapter, siteId } = options;
+	const plan = buildLogicalPlan(state, adapter.capabilities);
+	const dims = plan.groupByDimensions;
+	const extras = [];
+	if (!dims.includes("queryCanonical")) return extras;
+	const keywordsKey = adapter.tableKeyForDataset("keywords");
+	const t = adapter.schema[keywordsKey];
+	const table = adapter.tableRef(keywordsKey);
+	const whereParts = [];
+	if (adapter.siteIdColRef && siteId != null) whereParts.push(sql`${adapter.siteIdColRef(keywordsKey)} = ${siteId}`);
+	whereParts.push(sql`${adapter.dateColRef(keywordsKey)} >= ${plan.dateRange.startDate}`);
+	whereParts.push(sql`${adapter.dateColRef(keywordsKey)} <= ${plan.dateRange.endDate}`);
+	const whereExpr = whereParts.length > 0 ? sql`WHERE ${joinAnd(whereParts)}` : sql``;
+	const outerQueryCol = sql.raw("query");
+	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${t.query_canonical} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${t.query_canonical} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${t.query_canonical}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${t.query_canonical}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
+	extras.push({
+		key: "canonicalExtras",
+		sql: compiled.sql,
+		params: compiled.params
+	});
+	return extras;
+}
+const LOGICAL_DATASETS = {
+	pages: { dimensions: {
+		page: {
+			column: "url",
+			surfaces: ["api", "stored"]
+		},
+		date: {
+			column: "date",
+			surfaces: ["api", "stored"]
+		}
+	} },
+	keywords: { dimensions: {
+		query: {
+			column: "query",
+			surfaces: ["api", "stored"]
+		},
+		queryCanonical: {
+			column: "query_canonical",
+			surfaces: ["stored", "derived"]
+		},
+		date: {
+			column: "date",
+			surfaces: ["api", "stored"]
+		}
+	} },
+	page_keywords: { dimensions: {
+		page: {
+			column: "url",
+			surfaces: ["api", "stored"]
+		},
+		query: {
+			column: "query",
+			surfaces: ["api", "stored"]
+		},
+		queryCanonical: {
+			column: "query_canonical",
+			surfaces: ["stored", "derived"]
+		},
+		date: {
+			column: "date",
+			surfaces: ["api", "stored"]
+		}
+	} },
+	countries: { dimensions: {
+		country: {
+			column: "country",
+			surfaces: ["api", "stored"]
+		},
+		date: {
+			column: "date",
+			surfaces: ["api", "stored"]
+		}
+	} },
+	devices: { dimensions: {
+		device: {
+			column: "device",
+			surfaces: ["api", "stored"]
+		},
+		date: {
+			column: "date",
+			surfaces: ["api", "stored"]
+		}
+	} },
+	search_appearance: { dimensions: {
+		searchAppearance: {
+			column: "searchAppearance",
+			surfaces: ["api", "stored"]
+		},
+		date: {
+			column: "date",
+			surfaces: ["api", "stored"]
+		}
+	} }
+};
+function inferLogicalDataset(dimensions, filterDims = []) {
+	const allDims = new Set([...dimensions, ...filterDims]);
+	const has = (d) => allDims.has(d);
+	if (has("searchAppearance")) return "search_appearance";
+	if (has("page") && (has("query") || has("queryCanonical"))) return "page_keywords";
+	if (has("query") || has("queryCanonical")) return "keywords";
+	if (has("page")) return "pages";
+	if (has("country")) return "countries";
+	if (has("device")) return "devices";
+	return "keywords";
+}
+const METRIC_NAMES = [
+	"clicks",
+	"impressions",
+	"ctr",
+	"position"
+];
+function defaultSqliteUrlToPathExpr(col) {
+	return `CASE WHEN ${col} LIKE 'http%' THEN CASE WHEN INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/') > 0 THEN SUBSTR(${col}, INSTR(${col}, '://') + 2 + INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/')) ELSE '/' END ELSE ${col} END`;
+}
+function buildDimensionColumnMap(datasetToTableKey) {
+	const entries = Object.entries(datasetToTableKey).map(([dataset, tableKey]) => {
+		const dims = LOGICAL_DATASETS[dataset].dimensions;
+		return [tableKey, Object.fromEntries(Object.entries(dims).map(([dim, binding]) => [dim, binding?.column ?? dim]))];
+	});
+	return Object.fromEntries(entries);
+}
+function createSqlFragments(config) {
+	const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride } = config;
+	const DIM_COLUMN_MAP = buildDimensionColumnMap(datasetToTableKey);
+	function isMetricDimension(dim) {
+		return METRIC_NAMES.includes(dim);
+	}
+	function dimColumn(dim, table) {
+		return DIM_COLUMN_MAP[table]?.[dim] ?? dim;
+	}
+	function tableKeyForDataset(dataset) {
+		return datasetToTableKey[dataset];
+	}
+	function inferTable(dimensions, filterDims = []) {
+		return tableKeyForDataset(inferLogicalDataset(dimensions, filterDims));
+	}
+	const urlToPathExpr = urlToPathExprOverride ?? defaultSqliteUrlToPathExpr;
+	function colRef(tableKey, colName) {
+		const c = schema[tableKey][colName];
+		if (!c) throw new Error(`${tableLabel}: unknown column '${colName}' on ${tableKey}`);
+		return sql`${c}`;
+	}
+	function tableRef(tableKey) {
+		if (tableRefOverride) return tableRefOverride(tableKey);
+		return sql`${schema[tableKey]}`;
+	}
+	function dateColRef(tableKey) {
+		return colRef(tableKey, "date");
+	}
+	function siteIdColRef(tableKey) {
+		return colRef(tableKey, "site_id");
+	}
+	function dimExprSql(dim, tableKey) {
+		const colName = dimColumn(dim, tableKey);
+		if (dim === "page") return sql.raw(urlToPathExpr(colName));
+		return colRef(tableKey, colName);
+	}
+	function metricSql(metric, tableKey) {
+		const t = schema[tableKey];
+		switch (metric) {
+			case "clicks": return sql`SUM(${t.clicks})`;
+			case "impressions": return sql`SUM(${t.impressions})`;
+			case "ctr": return sql`CAST(SUM(${t.clicks}) AS ${sql.raw(metricCast)}) / NULLIF(SUM(${t.impressions}), 0)`;
+			case "position": return sql`SUM(${t.sum_position}) / NULLIF(SUM(${t.impressions}), 0) + 1`;
+		}
+	}
+	function havingPredicates(filters, tableKey) {
+		const preds = [];
+		for (const f of filters) {
+			const metric = f.dimension;
+			if (!isMetricDimension(metric)) continue;
+			const expr = metricSql(metric, tableKey);
+			const v = Number(f.expression);
+			switch (f.operator) {
+				case "metricGte":
+					preds.push(sql`${expr} >= ${v}`);
+					break;
+				case "metricGt":
+					preds.push(sql`${expr} > ${v}`);
+					break;
+				case "metricLte":
+					preds.push(sql`${expr} <= ${v}`);
+					break;
+				case "metricLt":
+					preds.push(sql`${expr} < ${v}`);
+					break;
+				case "metricBetween": {
+					const v2 = Number(f.expression2);
+					preds.push(sql`${expr} >= ${v} AND ${expr} <= ${v2}`);
+					break;
+				}
+			}
+		}
+		return preds;
+	}
+	function dimensionPredicates(filters, tableKey) {
+		const preds = [];
+		for (const f of filters) {
+			if (isMetricDimension(f.dimension)) continue;
+			if (f.dimension === "date") continue;
+			if (f.operator === "topLevel") continue;
+			const cRef = colRef(tableKey, dimColumn(f.dimension, tableKey));
+			const matchExpr = f.dimension === "page" ? dimExprSql(f.dimension, tableKey) : cRef;
+			switch (f.operator) {
+				case "equals":
+					preds.push(sql`${matchExpr} = ${f.expression}`);
+					break;
+				case "notEquals":
+					preds.push(sql`${matchExpr} != ${f.expression}`);
+					break;
+				case "contains":
+					preds.push(sql`${cRef} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					break;
+				case "notContains":
+					preds.push(sql`${cRef} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					break;
+				case "includingRegex":
+					preds.push(regexPredicate(cRef, f.expression, false));
+					break;
+				case "excludingRegex":
+					preds.push(regexPredicate(cRef, f.expression, true));
+					break;
+			}
+		}
+		return preds;
+	}
+	function topLevelPredicate(filters, tableKey) {
+		if (!filters.some((f) => f.operator === "topLevel")) return void 0;
+		const pathExpr = dimExprSql("page", tableKey);
+		return sql`LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
+	}
+	return {
+		METRIC_NAMES,
+		DIM_COLUMN_MAP,
+		isMetricDimension,
+		tableKeyForDataset,
+		dimColumn,
+		inferTable,
+		urlToPathExpr,
+		colRef,
+		tableRef,
+		dateColRef,
+		siteIdColRef: includeSiteId ? siteIdColRef : void 0,
+		dimExprSql,
+		metricSql,
+		havingPredicates,
+		dimensionPredicates,
+		topLevelPredicate
+	};
+}
+function createResolverAdapter(config) {
+	const runtime = createSqlFragments(config);
+	return {
+		METRIC_NAMES: runtime.METRIC_NAMES,
+		capabilities: config.capabilities,
+		schema: config.schema,
+		tableKeyForDataset: runtime.tableKeyForDataset,
+		inferTable: runtime.inferTable,
+		dimColumn: runtime.dimColumn,
+		isMetricDimension: runtime.isMetricDimension,
+		tableRef: runtime.tableRef,
+		dateColRef: runtime.dateColRef,
+		urlToPathExpr: runtime.urlToPathExpr,
+		siteIdColRef: runtime.siteIdColRef,
+		dimExprSql: runtime.dimExprSql,
+		metricSql: runtime.metricSql,
+		dimensionPredicates: runtime.dimensionPredicates,
+		havingPredicates: runtime.havingPredicates,
+		topLevelPredicate: runtime.topLevelPredicate,
+		compile: config.compile
+	};
+}
+const pgDialect = new PgDialect();
+new SQLiteAsyncDialect();
+function compilePg(query) {
+	const compiled = pgDialect.sqlToQuery(query);
+	return {
+		sql: compiled.sql,
+		params: compiled.params
+	};
+}
+const PG_BASE_CONFIG = {
+	schema: drizzleSchema,
+	datasetToTableKey: {
+		pages: "pages",
+		keywords: "keywords",
+		page_keywords: "page_keywords",
+		countries: "countries",
+		devices: "devices",
+		search_appearance: "search_appearance"
+	},
+	metricCast: "DOUBLE",
+	regexPredicate: (expr, pattern, negate) => negate ? sql`NOT regexp_matches(${expr}, ${pattern})` : sql`regexp_matches(${expr}, ${pattern})`,
+	urlToPathExpr: (col) => `CASE WHEN ${col} LIKE 'http%' THEN COALESCE(NULLIF(regexp_replace(${col}, '^https?://[^/]+', ''), ''), '/') ELSE ${col} END`,
+	includeSiteId: false,
+	compile: compilePg,
+	capabilities: {
+		regex: true,
+		comparisonJoin: true,
+		windowTotals: true
+	}
+};
+createResolverAdapter({
+	...PG_BASE_CONFIG,
+	tableLabel: "pg-resolver-adapter"
+});
+function createParquetResolverAdapter() {
+	return createResolverAdapter({
+		...PG_BASE_CONFIG,
+		tableLabel: "parquet-resolver-adapter",
+		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`)
+	});
+}
+const URL_PURGE_TABLES = ["pages", "page_keywords"];
+const MAX_DAY_BYTES = 100 * 1024 * 1024;
+const URL_COLUMNS = /* @__PURE__ */ new Set();
+for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);
+function normalizeRow(table, row) {
+	if (!URL_COLUMNS.has(`${table}:url`)) return row;
+	const url = row.url;
+	if (typeof url !== "string") return row;
+	const normalized = normalizeUrl(url);
+	if (normalized === url) return row;
+	return {
+		...row,
+		url: normalized
+	};
+}
+function createStorageEngine(opts) {
+	const { dataSource, manifestStore, codec, executor } = opts;
+	const defaultNow = opts.now ?? (() => Date.now());
+	async function writeDay(ctx, rows) {
+		if (!ctx.date) throw new Error("writeDay requires ctx.date");
+		const date = ctx.date;
+		const now = (ctx.now ?? defaultNow)();
+		const partition = dayPartition(date);
+		const searchType = ctx.searchType;
+		return manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition
+		}, async () => {
+			const superseding = (await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partitions: [partition]
+			})).filter((e) => inferSearchType(e) === inferSearchType({ searchType }));
+			const normalizedRows = rows.map((r) => normalizeRow(ctx.table, r));
+			const key = objectKey(ctx, ctx.table, partition, now, searchType);
+			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
+			let bytes = writtenBytes;
+			if (bytes === 0 && rowCount > 0 && dataSource.head) {
+				const probed = await dataSource.head(key);
+				if (probed) bytes = probed.bytes;
+			}
+			if (bytes > 104857600) {
+				await dataSource.delete([key]).catch(() => {});
+				throw new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+			}
+			const entry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: "raw",
+				...searchType !== void 0 ? { searchType } : {}
+			};
+			await manifestStore.registerVersion(entry, superseding);
+			await manifestStore.bumpWatermark({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table
+			}, date, now);
+		});
+	}
+	async function runSQL(opts) {
+		opts.signal?.throwIfAborted();
+		const entries = Object.entries(opts.fileSets);
+		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
+			return [name, (await manifestStore.listLive({
+				userId: opts.ctx.userId,
+				siteId: opts.ctx.siteId,
+				table: ref.table,
+				partitions: ref.partitions
+			})).map((e) => e.objectKey)];
+		}));
+		opts.signal?.throwIfAborted();
+		const fileKeys = {};
+		for (const [name, keys] of perSet) fileKeys[name] = keys;
+		const uniqueKeys = [...new Set(perSet.flatMap(([, keys]) => keys))];
+		let table = opts.table;
+		if (!table) {
+			if (new Set(entries.map(([, ref]) => ref.table)).size > 1) throw new Error("runSQL requires explicit ctx.table when fileSets reference multiple tables.");
+			table = entries[0]?.[1].table;
+		}
+		if (!table) throw new Error("runSQL requires at least one fileSet or an explicit table");
+		const result = await executor.execute({
+			sql: opts.sql,
+			params: opts.params ?? [],
+			fileKeys,
+			dataSource,
+			table,
+			signal: opts.signal
+		});
+		return {
+			rows: result.rows,
+			sql: result.sql,
+			objectKeys: uniqueKeys
+		};
+	}
+	async function query(ctx, state) {
+		const plan = buildLogicalPlan(state, { regex: true });
+		const table = ctx.table ?? plan.dataset;
+		const resolved = compileLogicalQueryPlan(plan, table);
+		return runSQL({
+			ctx: {
+				userId: ctx.userId,
+				siteId: ctx.siteId
+			},
+			table,
+			fileSets: { FILES: {
+				table,
+				partitions: resolved.partitions
+			} },
+			sql: resolved.sql,
+			params: resolved.params,
+			signal: ctx.signal
+		});
+	}
+	async function queryComparison(ctx, current, previous, filter) {
+		const adapter = createParquetResolverAdapter();
+		const currentPlan = buildLogicalPlan(current, adapter.capabilities);
+		const previousPlan = buildLogicalPlan(previous, adapter.capabilities);
+		if (currentPlan.dataset !== previousPlan.dataset) throw new Error(`queryComparison: current (${currentPlan.dataset}) and previous (${previousPlan.dataset}) must resolve to the same table`);
+		const table = ctx.table ?? currentPlan.dataset;
+		const comparison = resolveComparisonSQL(current, previous, {
+			adapter,
+			siteId: void 0
+		}, filter);
+		const totals = buildTotalsSql(current, {
+			adapter,
+			siteId: void 0
+		});
+		const fileSets = { FILES: {
+			table,
+			partitions: enumeratePartitions(currentPlan.dateRange.startDate < previousPlan.dateRange.startDate ? currentPlan.dateRange.startDate : previousPlan.dateRange.startDate, currentPlan.dateRange.endDate > previousPlan.dateRange.endDate ? currentPlan.dateRange.endDate : previousPlan.dateRange.endDate)
+		} };
+		const baseCtx = {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		};
+		const [main, count, totalsRow] = await Promise.all([
+			runSQL({
+				ctx: baseCtx,
+				table,
+				fileSets,
+				sql: comparison.sql,
+				params: comparison.params,
+				signal: ctx.signal
+			}),
+			runSQL({
+				ctx: baseCtx,
+				table,
+				fileSets,
+				sql: comparison.countSql,
+				params: comparison.countParams,
+				signal: ctx.signal
+			}),
+			runSQL({
+				ctx: baseCtx,
+				table,
+				fileSets,
+				sql: totals.sql,
+				params: totals.params,
+				signal: ctx.signal
+			})
+		]);
+		return {
+			rows: main.rows,
+			totalCount: Number(count.rows[0]?.total ?? 0),
+			totals: totalsRow.rows[0] ?? {}
+		};
+	}
+	async function queryExtras(ctx, state) {
+		const adapter = createParquetResolverAdapter();
+		const extras = buildExtrasQueries(state, {
+			adapter,
+			siteId: void 0
+		});
+		if (extras.length === 0) return [];
+		const plan = buildLogicalPlan(state, adapter.capabilities);
+		const table = ctx.table ?? plan.dataset;
+		const fileSets = { FILES: {
+			table,
+			partitions: enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate)
+		} };
+		const baseCtx = {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		};
+		const results = await Promise.all(extras.map((e) => runSQL({
+			ctx: baseCtx,
+			table,
+			fileSets,
+			sql: e.sql,
+			params: e.params,
+			signal: ctx.signal
+		})));
+		return extras.map((e, i) => ({
+			key: e.key,
+			rows: results[i].rows
+		}));
+	}
+	async function compactTiered(ctx, thresholds) {
+		return compactTieredImpl({
+			dataSource,
+			manifestStore,
+			codec
+		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
+	}
+	async function gcOrphans(ctx, graceMs) {
+		return gcOrphansImpl({
+			dataSource,
+			manifestStore
+		}, (ctx.now ?? defaultNow)(), graceMs, {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		});
+	}
+	async function purgeTenant(ctx) {
+		const prefix = tenantPrefix(ctx);
+		const keys = [];
+		const keyStream = dataSource.streamList ? dataSource.streamList(prefix) : async function* () {
+			for (const k of await dataSource.list(prefix)) yield k;
+		}();
+		for await (const key of keyStream) keys.push(key);
+		if (keys.length > 0) await dataSource.delete(keys);
+		const manifestResult = await manifestStore.purgeTenant({
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		});
+		return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			prefix,
+			objectsDeleted: keys.length,
+			entriesRemoved: manifestResult.entriesRemoved,
+			watermarksRemoved: manifestResult.watermarksRemoved,
+			syncStatesRemoved: manifestResult.syncStatesRemoved,
+			at: defaultNow()
+		};
+	}
+	async function purgeUrls(ctx, urls) {
+		const now = defaultNow();
+		const urlSet = new Set(urls);
+		let entriesRewritten = 0;
+		let rowsRemoved = 0;
+		let bytesAfter = 0;
+		if (urlSet.size === 0) return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			urlsRequested: 0,
+			entriesRewritten: 0,
+			rowsRemoved: 0,
+			bytesAfter: 0,
+			at: now
+		};
+		for (const table of URL_PURGE_TABLES) {
+			const entries = await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table
+			});
+			for (const entry of entries) await manifestStore.withLock({
+				userId: entry.userId,
+				siteId: entry.siteId,
+				table,
+				partition: entry.partition
+			}, async () => {
+				const rows = await codec.readRows({ table }, entry.objectKey, dataSource);
+				const kept = rows.filter((r) => typeof r.url !== "string" || !urlSet.has(r.url));
+				const removed = rows.length - kept.length;
+				if (removed === 0) return;
+				const searchType = entry.searchType;
+				const newKey = objectKey({
+					userId: entry.userId,
+					siteId: entry.siteId
+				}, table, entry.partition, now, searchType);
+				const { bytes, rowCount } = await codec.writeRows({ table }, kept, newKey, dataSource);
+				const newEntry = {
+					userId: entry.userId,
+					siteId: entry.siteId,
+					table,
+					partition: entry.partition,
+					objectKey: newKey,
+					rowCount,
+					bytes,
+					createdAt: now,
+					schemaVersion: entry.schemaVersion ?? currentSchemaVersion(table),
+					...entry.tier !== void 0 ? { tier: entry.tier } : {},
+					...searchType !== void 0 ? { searchType } : {}
+				};
+				await manifestStore.registerVersion(newEntry, [entry]);
+				entriesRewritten++;
+				rowsRemoved += removed;
+				bytesAfter += bytes;
+			});
+		}
+		return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			urlsRequested: urlSet.size,
+			entriesRewritten,
+			rowsRemoved,
+			bytesAfter,
+			at: now
+		};
+	}
+	return {
+		writeDay,
+		query,
+		queryComparison,
+		queryExtras,
+		runSQL,
+		compactTiered,
+		gcOrphans,
+		purgeTenant,
+		purgeUrls,
+		listLive: (filter) => manifestStore.listLive(filter),
+		listAll: (filter) => manifestStore.listAll(filter),
+		getWatermarks: (filter) => manifestStore.getWatermarks(filter),
+		getSyncStates: (filter) => manifestStore.getSyncStates(filter),
+		setSyncState: (scope, state, detail) => manifestStore.setSyncState(scope, state, detail),
+		readObject: (key) => dataSource.read(key)
+	};
+}
+function arrowToRows(result) {
+	const r = result;
+	const arr = Array.isArray(r) ? r : typeof r?.toArray === "function" ? r.toArray() : [];
+	if (!arr || arr.length === 0) return [];
+	if (typeof arr[0]?.toJSON === "function") return arr.map((r) => r.toJSON());
+	return arr;
+}
+const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
+let singleton = null;
+function bundles() {
+	return {
+		mvp: {
+			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm"),
+			mainWorker: null
+		},
+		eh: {
+			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm"),
+			mainWorker: null
+		}
+	};
+}
+async function initialize(opts) {
+	const logger = opts.verbose ? new ConsoleLogger() : new VoidLogger();
+	const db = await createDuckDB(bundles(), logger, NODE_RUNTIME);
+	await db.instantiate();
+	return {
+		db,
+		conn: db.connect()
+	};
+}
+function createNodeDuckDBHandle(opts = {}) {
+	if (!singleton) singleton = initialize(opts);
+	return {
+		async query(sql, params) {
+			const { conn } = await singleton;
+			if (!params || params.length === 0) return arrowToRows(conn.query(sql));
+			const stmt = conn.prepare(sql);
+			try {
+				return arrowToRows(stmt.query(...params));
+			} finally {
+				stmt.close();
+			}
+		},
+		async registerFileBuffer(name, bytes) {
+			const { db } = await singleton;
+			db.registerFileBuffer(name, bytes);
+		},
+		async copyFileToBuffer(name) {
+			const { db } = await singleton;
+			return db.copyFileToBuffer(name);
+		},
+		async dropFiles(names) {
+			const { db } = await singleton;
+			for (const name of names) {
+				try {
+					db.dropFile(name);
+				} catch {}
+				try {
+					unlinkSync(name);
+				} catch {}
+			}
+		},
+		makeTempPath(ext) {
+			return join(tmpdir(), `gscdump-${Math.random().toString(36).slice(2, 10)}.${ext}`);
+		}
+	};
+}
+function createFilesystemDataSource(opts) {
+	const root = resolve(opts.rootDir);
+	function pathFor(key) {
+		const resolved = resolve(root, key);
+		if (!resolved.startsWith(`${root}/`) && resolved !== root) throw new Error(`path escapes root: ${key}`);
+		return resolved;
+	}
+	return {
+		async read(key, range, signal) {
+			const bytes = await readFile(pathFor(key), { signal });
+			if (!range) return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+			const sliced = bytes.subarray(range.offset, range.offset + range.length);
+			return new Uint8Array(sliced.buffer, sliced.byteOffset, sliced.byteLength);
+		},
+		async write(key, bytes) {
+			const path = pathFor(key);
+			await mkdir(dirname(path), { recursive: true });
+			await writeFile(path, Buffer.from(bytes));
+		},
+		async delete(keys) {
+			await Promise.all(keys.map(async (k) => {
+				await rm(pathFor(k), { force: true });
+			}));
+		},
+		async list(prefix) {
+			const full = resolve(root, prefix);
+			const out = [];
+			await walk(full, out);
+			return out.map((p) => p.slice(root.length + 1));
+		},
+		async *streamList(prefix) {
+			const full = resolve(root, prefix);
+			for await (const p of walkStream(full)) yield p.slice(root.length + 1);
+		},
+		async head(key) {
+			return stat(pathFor(key)).then((s) => ({ bytes: s.size }), (err) => {
+				if (err.code === "ENOENT") return void 0;
+				throw err;
+			});
+		},
+		uri(key) {
+			return pathFor(key);
+		}
+	};
+}
+async function* walkStream(dir) {
+	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
+		if (err.code === "ENOENT") return [];
+		throw err;
+	});
+	for (const entry of entries) {
+		const p = join(dir, String(entry.name));
+		if (entry.isDirectory()) yield* walkStream(p);
+		else yield p;
+	}
+}
+async function walk(dir, out) {
+	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
+		if (err.code === "ENOENT") return [];
+		throw err;
+	});
+	for (const entry of entries) {
+		const p = join(dir, String(entry.name));
+		if (entry.isDirectory()) await walk(p, out);
+		else out.push(p);
+	}
+}
+function watermarkKey(w) {
+	return `${w.userId}|${w.siteId ?? ""}|${w.table}`;
+}
+function matchesWatermarkFilter(w, filter) {
+	if (w.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && w.table !== filter.table) return false;
+	return true;
+}
+function syncStateKey(s) {
+	return `${s.userId}|${s.siteId ?? ""}|${s.table}|${s.date}|${inferSearchType(s)}`;
+}
+function matchesSyncStateFilter(s, filter) {
+	if (s.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && s.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && s.table !== filter.table) return false;
+	if (filter.state !== void 0 && s.state !== filter.state) return false;
+	if (filter.searchType !== void 0 && inferSearchType(s) !== filter.searchType) return false;
+	return true;
+}
+function mergeSyncState(existing, scope, state, detail) {
+	const at = detail?.at ?? Date.now();
+	const attemptsBump = state === "inflight" ? 1 : 0;
+	if (!existing) return {
+		userId: scope.userId,
+		siteId: scope.siteId,
+		table: scope.table,
+		date: scope.date,
+		state,
+		updatedAt: at,
+		attempts: attemptsBump,
+		error: detail?.error,
+		...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
+	};
+	return {
+		...existing,
+		state,
+		updatedAt: at,
+		attempts: existing.attempts + attemptsBump,
+		error: state === "done" ? void 0 : detail?.error ?? existing.error
+	};
+}
+function matchesFilter(entry, filter) {
+	if (entry.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && entry.table !== filter.table) return false;
+	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
+	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	return true;
+}
+function lockFileFor(locksDir, scope) {
+	return join(locksDir, `${`${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_")}.lock`);
+}
+function createFilesystemManifestStore(opts) {
+	const manifestPath = resolve(opts.path);
+	const locksDir = join(dirname(manifestPath), "locks");
+	async function load() {
+		const content = await readFile(manifestPath, "utf8").catch((err) => {
+			if (err.code === "ENOENT") return null;
+			throw err;
+		});
+		if (content === null) return {
+			version: 1,
+			entries: []
+		};
+		const parsed = JSON.parse(content);
+		if (parsed.version !== 1) throw new Error(`unsupported manifest version ${parsed.version}`);
+		return parsed;
+	}
+	async function save(data) {
+		await mkdir(dirname(manifestPath), { recursive: true });
+		const tmp = `${manifestPath}.${randomBytes(6).toString("hex")}.tmp`;
+		await writeFile(tmp, JSON.stringify(data), "utf8");
+		await rename(tmp, manifestPath).catch(async (err) => {
+			await unlink(tmp).catch(() => {});
+			throw err;
+		});
+	}
+	const queue = [];
+	let running = false;
+	function enqueue(fn) {
+		return new Promise((resolvePromise, rejectPromise) => {
+			queue.push(async () => {
+				await fn().then(resolvePromise, rejectPromise);
+			});
+			drain();
+		});
+	}
+	async function drain() {
+		if (running) return;
+		running = true;
+		while (queue.length > 0) await queue.shift()().catch(() => {});
+		running = false;
+	}
+	function entryKey(e) {
+		return e.objectKey;
+	}
+	async function registerVersionsImpl(newEntries, superseding) {
+		const data = await load();
+		const supersededAt = newEntries[0]?.createdAt ?? Date.now();
+		const byKey = new Map(data.entries.map((e) => [entryKey(e), e]));
+		if (superseding) for (const s of superseding) {
+			const existing = byKey.get(entryKey(s));
+			if (existing && existing.retiredAt === void 0) byKey.set(entryKey(s), {
+				...existing,
+				retiredAt: supersededAt
+			});
+		}
+		for (const e of newEntries) byKey.set(entryKey(e), e);
+		data.entries = Array.from(byKey.values());
+		await save(data);
+	}
+	return {
+		async listLive(filter) {
+			return (await load()).entries.filter((e) => e.retiredAt === void 0 && matchesFilter(e, filter));
+		},
+		async listAll(filter) {
+			return (await load()).entries.filter((e) => matchesFilter(e, filter));
+		},
+		async registerVersion(entry, superseding) {
+			return enqueue(() => registerVersionsImpl([entry], superseding));
+		},
+		async registerVersions(entries, superseding) {
+			return enqueue(() => registerVersionsImpl(entries, superseding));
+		},
+		async listRetired(olderThan) {
+			return (await load()).entries.filter((e) => e.retiredAt !== void 0 && e.retiredAt <= olderThan);
+		},
+		async delete(toDelete) {
+			return enqueue(async () => {
+				const data = await load();
+				const toDeleteKeys = new Set(toDelete.map(entryKey));
+				data.entries = data.entries.filter((e) => !toDeleteKeys.has(entryKey(e)));
+				await save(data);
+			});
+		},
+		async getWatermarks(filter) {
+			return ((await load()).watermarks ?? []).filter((w) => matchesWatermarkFilter(w, filter));
+		},
+		async getSyncStates(filter) {
+			return ((await load()).syncStates ?? []).filter((s) => matchesSyncStateFilter(s, filter));
+		},
+		async setSyncState(scope, state, detail) {
+			return enqueue(async () => {
+				const data = await load();
+				const key = syncStateKey(scope);
+				const byKey = new Map((data.syncStates ?? []).map((s) => [syncStateKey(s), s]));
+				byKey.set(key, mergeSyncState(byKey.get(key), scope, state, detail));
+				data.syncStates = Array.from(byKey.values());
+				await save(data);
+			});
+		},
+		async withLock(scope, fn) {
+			await mkdir(locksDir, { recursive: true });
+			const path = lockFileFor(locksDir, scope);
+			await writeFile(path, "", { flag: "a" });
+			const release = await lock(path, {
+				realpath: false,
+				stale: 3e4,
+				retries: {
+					retries: 20,
+					minTimeout: 50,
+					maxTimeout: 500,
+					factor: 1.5
+				}
+			});
+			return await fn().finally(() => release().catch(() => {}));
+		},
+		async purgeTenant(filter) {
+			return enqueue(async () => {
+				const data = await load();
+				const matches = (r) => r.userId === filter.userId && (filter.siteId === void 0 || r.siteId === filter.siteId);
+				const before = {
+					entries: data.entries.length,
+					watermarks: (data.watermarks ?? []).length,
+					syncStates: (data.syncStates ?? []).length
+				};
+				data.entries = data.entries.filter((e) => !matches(e));
+				data.watermarks = (data.watermarks ?? []).filter((w) => !matches(w));
+				data.syncStates = (data.syncStates ?? []).filter((s) => !matches(s));
+				await save(data);
+				return {
+					entriesRemoved: before.entries - data.entries.length,
+					watermarksRemoved: before.watermarks - data.watermarks.length,
+					syncStatesRemoved: before.syncStates - data.syncStates.length
+				};
+			});
+		},
+		async bumpWatermark(scope, date, at) {
+			return enqueue(async () => {
+				const data = await load();
+				const key = watermarkKey(scope);
+				const byKey = new Map((data.watermarks ?? []).map((w) => [watermarkKey(w), w]));
+				const existing = byKey.get(key);
+				const nowMs = at ?? Date.now();
+				const next = existing ? {
+					...existing,
+					newestDateSynced: date > existing.newestDateSynced ? date : existing.newestDateSynced,
+					oldestDateSynced: date < existing.oldestDateSynced ? date : existing.oldestDateSynced,
+					lastSyncAt: nowMs > existing.lastSyncAt ? nowMs : existing.lastSyncAt
+				} : {
+					userId: scope.userId,
+					siteId: scope.siteId,
+					table: scope.table,
+					newestDateSynced: date,
+					oldestDateSynced: date,
+					lastSyncAt: nowMs
+				};
+				byKey.set(key, next);
+				data.watermarks = Array.from(byKey.values());
+				await save(data);
+			});
+		}
+	};
+}
+function createNodeHarness(opts) {
+	const dataDir = opts.dataDir;
+	const userId = opts.userId ?? "local";
+	const manifestFilename = opts.manifestFilename ?? "manifest.json";
+	const handle = createNodeDuckDBHandle();
+	const factory = { getDuckDB: async () => handle };
+	const dataSource = createFilesystemDataSource({ rootDir: dataDir });
+	const engine = createStorageEngine({
+		dataSource,
+		manifestStore: createFilesystemManifestStore({ path: path.join(dataDir, manifestFilename) }),
+		codec: createDuckDBCodec(factory),
+		executor: createDuckDBExecutor(factory)
+	});
+	async function runRawSql(runOpts) {
+		const result = await engine.runSQL({
+			ctx: {
+				userId,
+				siteId: encodeSiteId(runOpts.siteUrl)
+			},
+			table: runOpts.table,
+			fileSets: { FILES: { table: runOpts.table } },
+			sql: runOpts.sql,
+			params: runOpts.params ?? []
+		});
+		return {
+			rows: result.rows,
+			sql: result.sql,
+			keys: result.objectKeys
+		};
+	}
+	return {
+		engine,
+		dataSource,
+		dataDir,
+		userId,
+		siteIdFor: encodeSiteId,
+		runRawSql
+	};
+}
+export { createNodeHarness };