@gscdump/engine 0.9.2 → 0.10.0

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, G as monthPartition, H as inferLegacyTier, I as Watermark, J as quarterPartition, K as objectKey, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, U as inferSearchType, V as dayPartition, W as mondayOfWeek, X as CompactionThresholds, Y as weekPartition, Z as enumeratePartitions, _ as PurgeFilter, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, h as OptimizedQueryResult, i as DEFAULT_SEARCH_TYPE, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, q as quarterOfMonth, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeResult, w as QueryResult, x as QueryExecuteOptions, y as PurgeUrlsResult, z as WriteCtx } from "./_chunks/storage.mjs";
+import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, G as CompactionThresholds, H as inferLegacyTier, I as Watermark, K as enumeratePartitions, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, U as inferSearchType, V as dayPartition, W as objectKey, _ as PurgeFilter, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, h as OptimizedQueryResult, i as DEFAULT_SEARCH_TYPE, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeResult, w as QueryResult, x as QueryExecuteOptions, y as PurgeUrlsResult, z as WriteCtx } from "./_chunks/storage.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
 import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema } from "./_chunks/schema.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
@@ -9,4 +9,4 @@ declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, objectKey, page_keywords, pages, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow };

package/dist/index.mjs

@@ -1,11 +1,10 @@
 import { a as inferTable, c as countries, d as keywords, f as page_keywords, i as dimensionToColumn, l as devices, n as allTables, p as pages, r as currentSchemaVersion, s as TABLE_METADATA, t as SCHEMAS, u as drizzleSchema } from "./_chunks/schema.mjs";
-import { a as mondayOfWeek, c as quarterOfMonth, d as weekPartition, i as inferSearchType, l as quarterPartition, n as dayPartition, o as monthPartition, r as inferLegacyTier, s as objectKey, t as DEFAULT_SEARCH_TYPE, u as tenantPrefix } from "./_chunks/storage.mjs";
-import { a as compactTieredImpl, i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
-import { bindLiterals, formatLiteral, sqlEscape } from "./sql-bind.mjs";
-import { g as resolveComparisonSQL, m as buildTotalsSql, p as buildExtrasQueries, t as createParquetResolverAdapter, v as resolveToSQLOptimized } from "./_chunks/pg-adapter.mjs";
+import { i as inferSearchType, n as dayPartition, r as inferLegacyTier, s as objectKey, t as DEFAULT_SEARCH_TYPE } from "./_chunks/storage.mjs";
+import { i as substituteNamedFiles, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/compiler.mjs";
+import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
+import { a as createDuckDBExecutor, i as createDuckDBCodec, n as createStorageEngine, r as canonicalEmptyParquetSchema, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
 import { createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
-import { buildLogicalPlan } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump/normalize";
+import "./planner.mjs";
 function coerceRow(row) {
 	let mutated = null;
 	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
@@ -19,618 +18,4 @@ function coerceRows(rows) {
 	for (let i = 0; i < rows.length; i++) out[i] = coerceRow(rows[i]);
 	return out;
 }
-async function encodeBytes(db, table, rows) {
-	const inName = db.makeTempPath("json");
-	const outName = db.makeTempPath("parquet");
-	const jsonBytes = new TextEncoder().encode(JSON.stringify(rows));
-	const registered = [];
-	await db.registerFileBuffer(inName, jsonBytes);
-	registered.push(inName);
-	try {
-		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`;
-		await db.query(sql);
-		registered.push(outName);
-		return await db.copyFileToBuffer(outName);
-	} finally {
-		await db.dropFiles(registered);
-	}
-}
-async function decodeBytes(db, bytes, table) {
-	const name = db.makeTempPath("parquet");
-	await db.registerFileBuffer(name, bytes);
-	try {
-		return await db.query(`SELECT * ${dateReplaceClause(table)} FROM read_parquet('${sqlEscape(name)}')`);
-	} finally {
-		await db.dropFiles([name]);
-	}
-}
-function createDuckDBCodec(factory) {
-	return {
-		async writeRows(ctx, rows, key, dataSource) {
-			const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
-			await dataSource.write(key, bytes);
-			return {
-				bytes: bytes.byteLength,
-				rowCount: rows.length
-			};
-		},
-		async readRows(ctx, key, dataSource) {
-			return decodeBytes(await factory.getDuckDB(), await dataSource.read(key), ctx.table);
-		},
-		async compactRows(ctx, inputKeys, outputKey, dataSource) {
-			const db = await factory.getDuckDB();
-			if (inputKeys.length === 0) {
-				const bytes = await encodeBytes(db, ctx.table, []);
-				await dataSource.write(outputKey, bytes);
-				return {
-					bytes: bytes.byteLength,
-					rowCount: 0
-				};
-			}
-			const inputUris = inputKeys.map((k) => dataSource.uri?.(k));
-			if (inputUris.every((u) => u !== void 0)) {
-				const outName = db.makeTempPath("parquet");
-				const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
-				try {
-					await db.query(`COPY (SELECT * FROM read_parquet([${fileList}], union_by_name=true)) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
-					const bytes = await db.copyFileToBuffer(outName);
-					const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
-					const rowCount = Number(countRows[0]?.n ?? 0);
-					await dataSource.write(outputKey, bytes);
-					return {
-						bytes: bytes.byteLength,
-						rowCount
-					};
-				} finally {
-					await db.dropFiles([outName]);
-				}
-			}
-			const inputs = await Promise.all(inputKeys.map((k) => dataSource.read(k)));
-			const inNames = [];
-			const outName = db.makeTempPath("parquet");
-			const registered = [];
-			for (let i = 0; i < inputs.length; i++) {
-				const name = db.makeTempPath("parquet");
-				await db.registerFileBuffer(name, inputs[i]);
-				inNames.push(name);
-				registered.push(name);
-			}
-			try {
-				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
-				await db.query(`COPY (SELECT * FROM read_parquet([${fileList}], union_by_name = true)) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
-				registered.push(outName);
-				const bytes = await db.copyFileToBuffer(outName);
-				const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
-				const rowCount = Number(countRows[0]?.n ?? 0);
-				await dataSource.write(outputKey, bytes);
-				return {
-					bytes: bytes.byteLength,
-					rowCount
-				};
-			} finally {
-				await db.dropFiles(registered);
-			}
-		}
-	};
-}
-function rewriteEmptyFileSets(sql, placeholders, defaultTable, placeholderTables) {
-	let out = sql;
-	for (const [name, keys] of Object.entries(placeholders)) {
-		if (keys.length > 0) continue;
-		const emptyFallback = `(SELECT * FROM ${emptyTableSchema(placeholderTables?.[name] ?? defaultTable)} WHERE FALSE)`;
-		const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${name}\\}\\}\\s*(?:,\\s*union_by_name\\s*=\\s*true\\s*)?\\)`, "g");
-		out = out.replace(pattern, emptyFallback);
-	}
-	return out;
-}
-function createDuckDBExecutor(factory) {
-	return { async execute({ sql, params, fileKeys, placeholderTables, dataSource, table, signal }) {
-		signal?.throwIfAborted();
-		const db = await factory.getDuckDB();
-		const placeholders = {};
-		const registered = [];
-		for (const [name, keys] of Object.entries(fileKeys)) {
-			const resolved = [];
-			for (const key of keys) {
-				const uri = dataSource.uri?.(key);
-				if (uri !== void 0) resolved.push(uri);
-				else {
-					const bytes = await dataSource.read(key, void 0, signal);
-					await db.registerFileBuffer(key, bytes);
-					registered.push(key);
-					resolved.push(key);
-				}
-			}
-			placeholders[name] = resolved;
-		}
-		try {
-			signal?.throwIfAborted();
-			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
-			return {
-				rows: await db.query(finalSql, params),
-				sql: finalSql
-			};
-		} finally {
-			if (registered.length > 0) await db.dropFiles(registered);
-		}
-	} };
-}
-function emptyTableSchema(table) {
-	return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
-}
-function canonicalEmptyParquetSchema(table) {
-	return emptyTableSchema(table);
-}
-function dateReplaceClause(table) {
-	if (!table) return "";
-	const dateCols = SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
-	if (dateCols.length === 0) return "";
-	return `REPLACE (${dateCols.map((n) => `strftime(${n}, '%Y-%m-%d') AS ${n}`).join(", ")})`;
-}
-function columnList(table) {
-	return SCHEMAS[table].columns.map((c) => c.name).join(", ");
-}
-function placeholderValues(table) {
-	return `(${SCHEMAS[table].columns.map((c) => defaultForType(c.type)).join(", ")})`;
-}
-function defaultForType(t) {
-	if (t === "VARCHAR") return "''";
-	if (t === "DATE") return "DATE '1970-01-01'";
-	if (t === "INTEGER" || t === "BIGINT") return "0";
-	if (t === "DOUBLE") return "CAST(0 AS DOUBLE)";
-	return "NULL";
-}
-function columnsJson(table) {
-	return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
-}
-const VERSION_RE = /__v(\d+)\.parquet$/;
-function parseLockScope(key) {
-	const match = VERSION_RE.exec(key);
-	if (!match) return void 0;
-	const parts = key.slice(0, match.index).split("/");
-	if (parts.length < 4) return void 0;
-	const userPart = parts[0];
-	if (!userPart.startsWith("u_")) return void 0;
-	const userId = userPart.slice(2);
-	const partition = parts.slice(-2).join("/");
-	const table = parts[parts.length - 3];
-	return {
-		userId,
-		siteId: parts.length >= 5 ? parts.slice(1, -3).join("/") : void 0,
-		table,
-		partition
-	};
-}
-async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
-	const cutoff = now - graceMs;
-	const retired = await deps.manifestStore.listRetired(cutoff);
-	if (retired.length > 0) {
-		await deps.dataSource.delete(retired.map((e) => e.objectKey));
-		await deps.manifestStore.delete(retired);
-	}
-	let sweptOrphans = 0;
-	if (opts.userId) {
-		const prefix = tenantPrefix({
-			userId: opts.userId,
-			siteId: opts.siteId
-		});
-		const knownEntries = await deps.manifestStore.listAll({
-			userId: opts.userId,
-			siteId: opts.siteId
-		});
-		const knownSet = new Set(knownEntries.map((e) => e.objectKey));
-		const orphans = [];
-		const keyStream = deps.dataSource.streamList ? deps.dataSource.streamList(prefix) : async function* () {
-			const all = await deps.dataSource.list(prefix);
-			for (const k of all) yield k;
-		}();
-		for await (const key of keyStream) {
-			if (knownSet.has(key)) continue;
-			const match = VERSION_RE.exec(key);
-			if (!match) continue;
-			if (Number(match[1]) <= cutoff) orphans.push(key);
-		}
-		const byScope = /* @__PURE__ */ new Map();
-		for (const key of orphans) {
-			const scope = parseLockScope(key);
-			if (!scope) continue;
-			const sk = `${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`;
-			const bucket = byScope.get(sk) ?? {
-				scope,
-				keys: []
-			};
-			bucket.keys.push(key);
-			byScope.set(sk, bucket);
-		}
-		for (const { scope, keys } of byScope.values()) await deps.manifestStore.withLock(scope, async () => {
-			const known = await deps.manifestStore.listAll({
-				userId: scope.userId,
-				siteId: scope.siteId,
-				table: scope.table,
-				partitions: [scope.partition]
-			});
-			const knownInScope = new Set(known.map((e) => e.objectKey));
-			const stillOrphans = keys.filter((k) => !knownInScope.has(k));
-			if (stillOrphans.length > 0) {
-				await deps.dataSource.delete(stillOrphans);
-				sweptOrphans += stillOrphans.length;
-			}
-		});
-	}
-	return { deleted: retired.length + sweptOrphans };
-}
-const URL_PURGE_TABLES = ["pages", "page_keywords"];
-const MAX_DAY_BYTES = 100 * 1024 * 1024;
-const URL_COLUMNS = /* @__PURE__ */ new Set();
-for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);
-function normalizeRow(table, row) {
-	if (!URL_COLUMNS.has(`${table}:url`)) return row;
-	const url = row.url;
-	if (typeof url !== "string") return row;
-	const normalized = normalizeUrl(url);
-	if (normalized === url) return row;
-	return {
-		...row,
-		url: normalized
-	};
-}
-function createStorageEngine(opts) {
-	const { dataSource, manifestStore, codec, executor } = opts;
-	const defaultNow = opts.now ?? (() => Date.now());
-	async function writeDay(ctx, rows) {
-		if (!ctx.date) throw new Error("writeDay requires ctx.date");
-		const date = ctx.date;
-		const now = (ctx.now ?? defaultNow)();
-		const partition = dayPartition(date);
-		const searchType = ctx.searchType;
-		return manifestStore.withLock({
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			table: ctx.table,
-			partition
-		}, async () => {
-			const superseding = (await manifestStore.listLive({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partitions: [partition]
-			})).filter((e) => inferSearchType(e) === inferSearchType({ searchType }));
-			const normalizedRows = rows.map((r) => normalizeRow(ctx.table, r));
-			const key = objectKey(ctx, ctx.table, partition, now, searchType);
-			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
-			let bytes = writtenBytes;
-			if (bytes === 0 && rowCount > 0 && dataSource.head) {
-				const probed = await dataSource.head(key);
-				if (probed) bytes = probed.bytes;
-			}
-			if (bytes > 104857600) {
-				await dataSource.delete([key]).catch(() => {});
-				throw new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
-			}
-			const entry = {
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partition,
-				objectKey: key,
-				rowCount,
-				bytes,
-				createdAt: now,
-				schemaVersion: currentSchemaVersion(ctx.table),
-				tier: "raw",
-				...searchType !== void 0 ? { searchType } : {}
-			};
-			await manifestStore.registerVersion(entry, superseding);
-			await manifestStore.bumpWatermark({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table
-			}, date, now);
-		});
-	}
-	async function runSQL(opts) {
-		opts.signal?.throwIfAborted();
-		const entries = Object.entries(opts.fileSets);
-		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
-			return [name, (await manifestStore.listLive({
-				userId: opts.ctx.userId,
-				siteId: opts.ctx.siteId,
-				table: ref.table,
-				partitions: ref.partitions
-			})).map((e) => e.objectKey)];
-		}));
-		opts.signal?.throwIfAborted();
-		const fileKeys = {};
-		for (const [name, keys] of perSet) fileKeys[name] = keys;
-		const uniqueKeys = [...new Set(perSet.flatMap(([, keys]) => keys))];
-		let table = opts.table;
-		if (!table) {
-			if (new Set(entries.map(([, ref]) => ref.table)).size > 1) throw new Error("runSQL requires explicit ctx.table when fileSets reference multiple tables.");
-			table = entries[0]?.[1].table;
-		}
-		if (!table) throw new Error("runSQL requires at least one fileSet or an explicit table");
-		const placeholderTables = {};
-		for (const [name, ref] of entries) placeholderTables[name] = ref.table;
-		const result = await executor.execute({
-			sql: opts.sql,
-			params: opts.params ?? [],
-			fileKeys,
-			placeholderTables,
-			dataSource,
-			table,
-			signal: opts.signal
-		});
-		return {
-			rows: result.rows,
-			sql: result.sql,
-			objectKeys: uniqueKeys
-		};
-	}
-	async function query(ctx, state) {
-		const plan = buildLogicalPlan(state, { regex: true });
-		const table = ctx.table ?? plan.dataset;
-		const resolved = compileLogicalQueryPlan(plan, table);
-		return runSQL({
-			ctx: {
-				userId: ctx.userId,
-				siteId: ctx.siteId
-			},
-			table,
-			fileSets: { FILES: {
-				table,
-				partitions: resolved.partitions
-			} },
-			sql: resolved.sql,
-			params: resolved.params,
-			signal: ctx.signal
-		});
-	}
-	async function queryComparison(ctx, current, previous, filter) {
-		const adapter = createParquetResolverAdapter();
-		const currentPlan = buildLogicalPlan(current, adapter.capabilities);
-		const previousPlan = buildLogicalPlan(previous, adapter.capabilities);
-		if (currentPlan.dataset !== previousPlan.dataset) throw new Error(`queryComparison: current (${currentPlan.dataset}) and previous (${previousPlan.dataset}) must resolve to the same table`);
-		const table = ctx.table ?? currentPlan.dataset;
-		const comparison = resolveComparisonSQL(current, previous, {
-			adapter,
-			siteId: void 0
-		}, filter);
-		const totals = buildTotalsSql(current, {
-			adapter,
-			siteId: void 0
-		});
-		const fileSets = { FILES: {
-			table,
-			partitions: enumeratePartitions(currentPlan.dateRange.startDate < previousPlan.dateRange.startDate ? currentPlan.dateRange.startDate : previousPlan.dateRange.startDate, currentPlan.dateRange.endDate > previousPlan.dateRange.endDate ? currentPlan.dateRange.endDate : previousPlan.dateRange.endDate)
-		} };
-		const baseCtx = {
-			userId: ctx.userId,
-			siteId: ctx.siteId
-		};
-		const [main, count, totalsRow] = await Promise.all([
-			runSQL({
-				ctx: baseCtx,
-				table,
-				fileSets,
-				sql: comparison.sql,
-				params: comparison.params,
-				signal: ctx.signal
-			}),
-			runSQL({
-				ctx: baseCtx,
-				table,
-				fileSets,
-				sql: comparison.countSql,
-				params: comparison.countParams,
-				signal: ctx.signal
-			}),
-			runSQL({
-				ctx: baseCtx,
-				table,
-				fileSets,
-				sql: totals.sql,
-				params: totals.params,
-				signal: ctx.signal
-			})
-		]);
-		return {
-			rows: main.rows,
-			totalCount: Number(count.rows[0]?.total ?? 0),
-			totals: totalsRow.rows[0] ?? {}
-		};
-	}
-	async function queryOptimized(ctx, state) {
-		const adapter = createParquetResolverAdapter();
-		const plan = buildLogicalPlan(state, adapter.capabilities);
-		const table = ctx.table ?? plan.dataset;
-		const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
-		const { sql, params } = resolveToSQLOptimized(state, {
-			adapter,
-			siteId: void 0
-		});
-		const result = await runSQL({
-			ctx: {
-				userId: ctx.userId,
-				siteId: ctx.siteId
-			},
-			table,
-			fileSets: { FILES: {
-				table,
-				partitions
-			} },
-			sql,
-			params,
-			signal: ctx.signal
-		});
-		const firstRow = result.rows[0];
-		const totalCount = Number(firstRow?.totalCount ?? 0);
-		const totals = {
-			clicks: Number(firstRow?.totalClicks ?? 0),
-			impressions: Number(firstRow?.totalImpressions ?? 0),
-			ctr: Number(firstRow?.totalCtr ?? 0),
-			position: Number(firstRow?.totalPosition ?? 0)
-		};
-		return {
-			rows: result.rows.map((r) => {
-				const { totalCount: _tc, totalClicks: _tcl, totalImpressions: _ti, totalCtr: _tr, totalPosition: _tp, ...rest } = r;
-				return rest;
-			}),
-			totalCount,
-			totals
-		};
-	}
-	async function queryExtras(ctx, state) {
-		const adapter = createParquetResolverAdapter();
-		const extras = buildExtrasQueries(state, {
-			adapter,
-			siteId: void 0
-		});
-		if (extras.length === 0) return [];
-		const plan = buildLogicalPlan(state, adapter.capabilities);
-		const table = ctx.table ?? plan.dataset;
-		const fileSets = { FILES: {
-			table,
-			partitions: enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate)
-		} };
-		const baseCtx = {
-			userId: ctx.userId,
-			siteId: ctx.siteId
-		};
-		const results = await Promise.all(extras.map((e) => runSQL({
-			ctx: baseCtx,
-			table,
-			fileSets,
-			sql: e.sql,
-			params: e.params,
-			signal: ctx.signal
-		})));
-		return extras.map((e, i) => ({
-			key: e.key,
-			rows: results[i].rows
-		}));
-	}
-	async function compactTiered(ctx, thresholds) {
-		return compactTieredImpl({
-			dataSource,
-			manifestStore,
-			codec
-		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
-	}
-	async function gcOrphans(ctx, graceMs) {
-		return gcOrphansImpl({
-			dataSource,
-			manifestStore
-		}, (ctx.now ?? defaultNow)(), graceMs, {
-			userId: ctx.userId,
-			siteId: ctx.siteId
-		});
-	}
-	async function purgeTenant(ctx) {
-		const prefix = tenantPrefix(ctx);
-		const keys = [];
-		const keyStream = dataSource.streamList ? dataSource.streamList(prefix) : async function* () {
-			for (const k of await dataSource.list(prefix)) yield k;
-		}();
-		for await (const key of keyStream) keys.push(key);
-		if (keys.length > 0) await dataSource.delete(keys);
-		const manifestResult = await manifestStore.purgeTenant({
-			userId: ctx.userId,
-			siteId: ctx.siteId
-		});
-		return {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			prefix,
-			objectsDeleted: keys.length,
-			entriesRemoved: manifestResult.entriesRemoved,
-			watermarksRemoved: manifestResult.watermarksRemoved,
-			syncStatesRemoved: manifestResult.syncStatesRemoved,
-			at: defaultNow()
-		};
-	}
-	async function purgeUrls(ctx, urls) {
-		const now = defaultNow();
-		const urlSet = new Set(urls);
-		let entriesRewritten = 0;
-		let rowsRemoved = 0;
-		let bytesAfter = 0;
-		if (urlSet.size === 0) return {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			urlsRequested: 0,
-			entriesRewritten: 0,
-			rowsRemoved: 0,
-			bytesAfter: 0,
-			at: now
-		};
-		for (const table of URL_PURGE_TABLES) {
-			const entries = await manifestStore.listLive({
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table
-			});
-			for (const entry of entries) await manifestStore.withLock({
-				userId: entry.userId,
-				siteId: entry.siteId,
-				table,
-				partition: entry.partition
-			}, async () => {
-				const rows = await codec.readRows({ table }, entry.objectKey, dataSource);
-				const kept = rows.filter((r) => typeof r.url !== "string" || !urlSet.has(r.url));
-				const removed = rows.length - kept.length;
-				if (removed === 0) return;
-				const searchType = entry.searchType;
-				const newKey = objectKey({
-					userId: entry.userId,
-					siteId: entry.siteId
-				}, table, entry.partition, now, searchType);
-				const { bytes, rowCount } = await codec.writeRows({ table }, kept, newKey, dataSource);
-				const newEntry = {
-					userId: entry.userId,
-					siteId: entry.siteId,
-					table,
-					partition: entry.partition,
-					objectKey: newKey,
-					rowCount,
-					bytes,
-					createdAt: now,
-					schemaVersion: entry.schemaVersion ?? currentSchemaVersion(table),
-					...entry.tier !== void 0 ? { tier: entry.tier } : {},
-					...searchType !== void 0 ? { searchType } : {}
-				};
-				await manifestStore.registerVersion(newEntry, [entry]);
-				entriesRewritten++;
-				rowsRemoved += removed;
-				bytesAfter += bytes;
-			});
-		}
-		return {
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			urlsRequested: urlSet.size,
-			entriesRewritten,
-			rowsRemoved,
-			bytesAfter,
-			at: now
-		};
-	}
-	return {
-		writeDay,
-		query,
-		queryComparison,
-		queryExtras,
-		queryOptimized,
-		runSQL,
-		compactTiered,
-		gcOrphans,
-		purgeTenant,
-		purgeUrls,
-		listLive: (filter) => manifestStore.listLive(filter),
-		listAll: (filter) => manifestStore.listAll(filter),
-		getWatermarks: (filter) => manifestStore.getWatermarks(filter),
-		getSyncStates: (filter) => manifestStore.getSyncStates(filter),
-		setSyncState: (scope, state, detail) => manifestStore.setSyncState(scope, state, detail),
-		readObject: (key) => dataSource.read(key)
-	};
-}
-export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, SCHEMAS, TABLE_METADATA, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };
+export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, SCHEMAS, TABLE_METADATA, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, objectKey, page_keywords, pages, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow };

package/dist/planner.d.mts

@@ -1,3 +1,3 @@
-import { Z as enumeratePartitions } from "./_chunks/storage.mjs";
+import { K as enumeratePartitions } from "./_chunks/storage.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, r as compileLogicalQueryPlan, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/dist/planner.mjs

@@ -1,2 +1,2 @@
-import { i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
+import { i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/compiler.mjs";
 export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };