@gscdump/engine 0.7.5 → 0.8.0

package/dist/index.mjs

@@ -1,11 +1,11 @@
 import { a as inferTable, c as countries, d as keywords, f as page_keywords, i as dimensionToColumn, l as devices, n as allTables, p as pages, r as currentSchemaVersion, s as TABLE_METADATA, t as SCHEMAS, u as drizzleSchema } from "./_chunks/schema.mjs";
-import { a as mondayOfWeek, c as quarterOfMonth, d as weekPartition, i as inferSearchType, l as quarterPartition, n as dayPartition, o as monthPartition, r as inferLegacyTier, s as objectKey, t as DEFAULT_SEARCH_TYPE } from "./_chunks/storage.mjs";
-import { i as substituteNamedFiles, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/compiler.mjs";
-import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { a as createDuckDBExecutor, i as createDuckDBCodec, n as createStorageEngine, r as canonicalEmptyParquetSchema, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
-import { createInspectionStoreSqlite, inspectionSqliteKey } from "./entities.mjs";
+import { a as mondayOfWeek, c as quarterOfMonth, d as weekPartition, i as inferSearchType, l as quarterPartition, n as dayPartition, o as monthPartition, r as inferLegacyTier, s as objectKey, t as DEFAULT_SEARCH_TYPE, u as tenantPrefix } from "./_chunks/storage.mjs";
+import { a as compactTieredImpl, i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
+import { bindLiterals, formatLiteral, sqlEscape } from "./sql-bind.mjs";
+import { g as resolveComparisonSQL, m as buildTotalsSql, p as buildExtrasQueries, t as createParquetResolverAdapter } from "./_chunks/pg-adapter.mjs";
 import { createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
-import "./planner.mjs";
+import { buildLogicalPlan } from "gscdump/query/plan";
+import { normalizeUrl } from "gscdump/normalize";
 function coerceRow(row) {
 	let mutated = null;
 	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
@@ -19,4 +19,574 @@ function coerceRows(rows) {
 	for (let i = 0; i < rows.length; i++) out[i] = coerceRow(rows[i]);
 	return out;
 }
-export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, SCHEMAS, TABLE_METADATA, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createInspectionStoreSqlite, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, inspectionSqliteKey, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };
+async function encodeBytes(db, table, rows) {
+	const inName = db.makeTempPath("json");
+	const outName = db.makeTempPath("parquet");
+	const jsonBytes = new TextEncoder().encode(JSON.stringify(rows));
+	const registered = [];
+	await db.registerFileBuffer(inName, jsonBytes);
+	registered.push(inName);
+	try {
+		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`;
+		await db.query(sql);
+		registered.push(outName);
+		return await db.copyFileToBuffer(outName);
+	} finally {
+		await db.dropFiles(registered);
+	}
+}
+async function decodeBytes(db, bytes, table) {
+	const name = db.makeTempPath("parquet");
+	await db.registerFileBuffer(name, bytes);
+	try {
+		return await db.query(`SELECT * ${dateReplaceClause(table)} FROM read_parquet('${sqlEscape(name)}')`);
+	} finally {
+		await db.dropFiles([name]);
+	}
+}
+function createDuckDBCodec(factory) {
+	return {
+		async writeRows(ctx, rows, key, dataSource) {
+			const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
+			await dataSource.write(key, bytes);
+			return {
+				bytes: bytes.byteLength,
+				rowCount: rows.length
+			};
+		},
+		async readRows(ctx, key, dataSource) {
+			return decodeBytes(await factory.getDuckDB(), await dataSource.read(key), ctx.table);
+		},
+		async compactRows(ctx, inputKeys, outputKey, dataSource) {
+			const db = await factory.getDuckDB();
+			if (inputKeys.length === 0) {
+				const bytes = await encodeBytes(db, ctx.table, []);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount: 0
+				};
+			}
+			const inputUris = inputKeys.map((k) => dataSource.uri?.(k));
+			if (inputUris.every((u) => u !== void 0)) {
+				const outName = db.makeTempPath("parquet");
+				const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
+				try {
+					await db.query(`COPY (SELECT * FROM read_parquet([${fileList}], union_by_name=true)) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
+					const bytes = await db.copyFileToBuffer(outName);
+					const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
+					const rowCount = Number(countRows[0]?.n ?? 0);
+					await dataSource.write(outputKey, bytes);
+					return {
+						bytes: bytes.byteLength,
+						rowCount
+					};
+				} finally {
+					await db.dropFiles([outName]);
+				}
+			}
+			const inputs = await Promise.all(inputKeys.map((k) => dataSource.read(k)));
+			const inNames = [];
+			const outName = db.makeTempPath("parquet");
+			const registered = [];
+			for (let i = 0; i < inputs.length; i++) {
+				const name = db.makeTempPath("parquet");
+				await db.registerFileBuffer(name, inputs[i]);
+				inNames.push(name);
+				registered.push(name);
+			}
+			try {
+				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
+				await db.query(`COPY (SELECT * FROM read_parquet([${fileList}], union_by_name = true)) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
+				registered.push(outName);
+				const bytes = await db.copyFileToBuffer(outName);
+				const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
+				const rowCount = Number(countRows[0]?.n ?? 0);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount
+				};
+			} finally {
+				await db.dropFiles(registered);
+			}
+		}
+	};
+}
+function rewriteEmptyFileSets(sql, placeholders, table) {
+	const emptyFallback = `(SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE)`;
+	let out = sql;
+	for (const [name, keys] of Object.entries(placeholders)) {
+		if (keys.length > 0) continue;
+		const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${name}\\}\\}\\s*(?:,\\s*union_by_name\\s*=\\s*true\\s*)?\\)`, "g");
+		out = out.replace(pattern, emptyFallback);
+	}
+	return out;
+}
+function createDuckDBExecutor(factory) {
+	return { async execute({ sql, params, fileKeys, dataSource, table, signal }) {
+		signal?.throwIfAborted();
+		const db = await factory.getDuckDB();
+		const placeholders = {};
+		const registered = [];
+		for (const [name, keys] of Object.entries(fileKeys)) {
+			const resolved = [];
+			for (const key of keys) {
+				const uri = dataSource.uri?.(key);
+				if (uri !== void 0) resolved.push(uri);
+				else {
+					const bytes = await dataSource.read(key, void 0, signal);
+					await db.registerFileBuffer(key, bytes);
+					registered.push(key);
+					resolved.push(key);
+				}
+			}
+			placeholders[name] = resolved;
+		}
+		try {
+			signal?.throwIfAborted();
+			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table), placeholders);
+			return {
+				rows: await db.query(finalSql, params),
+				sql: finalSql
+			};
+		} finally {
+			if (registered.length > 0) await db.dropFiles(registered);
+		}
+	} };
+}
+function emptyTableSchema(table) {
+	return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
+}
+function canonicalEmptyParquetSchema(table) {
+	return emptyTableSchema(table);
+}
+function dateReplaceClause(table) {
+	if (!table) return "";
+	const dateCols = SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
+	if (dateCols.length === 0) return "";
+	return `REPLACE (${dateCols.map((n) => `strftime(${n}, '%Y-%m-%d') AS ${n}`).join(", ")})`;
+}
+function columnList(table) {
+	return SCHEMAS[table].columns.map((c) => c.name).join(", ");
+}
+function placeholderValues(table) {
+	return `(${SCHEMAS[table].columns.map((c) => defaultForType(c.type)).join(", ")})`;
+}
+function defaultForType(t) {
+	if (t === "VARCHAR") return "''";
+	if (t === "DATE") return "DATE '1970-01-01'";
+	if (t === "INTEGER" || t === "BIGINT") return "0";
+	if (t === "DOUBLE") return "CAST(0 AS DOUBLE)";
+	return "NULL";
+}
+function columnsJson(table) {
+	return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
+}
+const VERSION_RE = /__v(\d+)\.parquet$/;
+function parseLockScope(key) {
+	const match = VERSION_RE.exec(key);
+	if (!match) return void 0;
+	const parts = key.slice(0, match.index).split("/");
+	if (parts.length < 4) return void 0;
+	const userPart = parts[0];
+	if (!userPart.startsWith("u_")) return void 0;
+	const userId = userPart.slice(2);
+	const partition = parts.slice(-2).join("/");
+	const table = parts[parts.length - 3];
+	return {
+		userId,
+		siteId: parts.length >= 5 ? parts.slice(1, -3).join("/") : void 0,
+		table,
+		partition
+	};
+}
+async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
+	const cutoff = now - graceMs;
+	const retired = await deps.manifestStore.listRetired(cutoff);
+	if (retired.length > 0) {
+		await deps.dataSource.delete(retired.map((e) => e.objectKey));
+		await deps.manifestStore.delete(retired);
+	}
+	let sweptOrphans = 0;
+	if (opts.userId) {
+		const prefix = tenantPrefix({
+			userId: opts.userId,
+			siteId: opts.siteId
+		});
+		const knownEntries = await deps.manifestStore.listAll({
+			userId: opts.userId,
+			siteId: opts.siteId
+		});
+		const knownSet = new Set(knownEntries.map((e) => e.objectKey));
+		const orphans = [];
+		const keyStream = deps.dataSource.streamList ? deps.dataSource.streamList(prefix) : async function* () {
+			const all = await deps.dataSource.list(prefix);
+			for (const k of all) yield k;
+		}();
+		for await (const key of keyStream) {
+			if (knownSet.has(key)) continue;
+			const match = VERSION_RE.exec(key);
+			if (!match) continue;
+			if (Number(match[1]) <= cutoff) orphans.push(key);
+		}
+		const byScope = /* @__PURE__ */ new Map();
+		for (const key of orphans) {
+			const scope = parseLockScope(key);
+			if (!scope) continue;
+			const sk = `${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`;
+			const bucket = byScope.get(sk) ?? {
+				scope,
+				keys: []
+			};
+			bucket.keys.push(key);
+			byScope.set(sk, bucket);
+		}
+		for (const { scope, keys } of byScope.values()) await deps.manifestStore.withLock(scope, async () => {
+			const known = await deps.manifestStore.listAll({
+				userId: scope.userId,
+				siteId: scope.siteId,
+				table: scope.table,
+				partitions: [scope.partition]
+			});
+			const knownInScope = new Set(known.map((e) => e.objectKey));
+			const stillOrphans = keys.filter((k) => !knownInScope.has(k));
+			if (stillOrphans.length > 0) {
+				await deps.dataSource.delete(stillOrphans);
+				sweptOrphans += stillOrphans.length;
+			}
+		});
+	}
+	return { deleted: retired.length + sweptOrphans };
+}
+const URL_PURGE_TABLES = ["pages", "page_keywords"];
+const MAX_DAY_BYTES = 100 * 1024 * 1024;
+const URL_COLUMNS = /* @__PURE__ */ new Set();
+for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);
+function normalizeRow(table, row) {
+	if (!URL_COLUMNS.has(`${table}:url`)) return row;
+	const url = row.url;
+	if (typeof url !== "string") return row;
+	const normalized = normalizeUrl(url);
+	if (normalized === url) return row;
+	return {
+		...row,
+		url: normalized
+	};
+}
+function createStorageEngine(opts) {
+	const { dataSource, manifestStore, codec, executor } = opts;
+	const defaultNow = opts.now ?? (() => Date.now());
+	async function writeDay(ctx, rows) {
+		if (!ctx.date) throw new Error("writeDay requires ctx.date");
+		const date = ctx.date;
+		const now = (ctx.now ?? defaultNow)();
+		const partition = dayPartition(date);
+		const searchType = ctx.searchType;
+		return manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition
+		}, async () => {
+			const superseding = (await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partitions: [partition]
+			})).filter((e) => inferSearchType(e) === inferSearchType({ searchType }));
+			const normalizedRows = rows.map((r) => normalizeRow(ctx.table, r));
+			const key = objectKey(ctx, ctx.table, partition, now, searchType);
+			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
+			let bytes = writtenBytes;
+			if (bytes === 0 && rowCount > 0 && dataSource.head) {
+				const probed = await dataSource.head(key);
+				if (probed) bytes = probed.bytes;
+			}
+			if (bytes > 104857600) {
+				await dataSource.delete([key]).catch(() => {});
+				throw new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+			}
+			const entry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: "raw",
+				...searchType !== void 0 ? { searchType } : {}
+			};
+			await manifestStore.registerVersion(entry, superseding);
+			await manifestStore.bumpWatermark({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table
+			}, date, now);
+		});
+	}
+	async function runSQL(opts) {
+		opts.signal?.throwIfAborted();
+		const entries = Object.entries(opts.fileSets);
+		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
+			return [name, (await manifestStore.listLive({
+				userId: opts.ctx.userId,
+				siteId: opts.ctx.siteId,
+				table: ref.table,
+				partitions: ref.partitions
+			})).map((e) => e.objectKey)];
+		}));
+		opts.signal?.throwIfAborted();
+		const fileKeys = {};
+		for (const [name, keys] of perSet) fileKeys[name] = keys;
+		const uniqueKeys = [...new Set(perSet.flatMap(([, keys]) => keys))];
+		let table = opts.table;
+		if (!table) {
+			if (new Set(entries.map(([, ref]) => ref.table)).size > 1) throw new Error("runSQL requires explicit ctx.table when fileSets reference multiple tables.");
+			table = entries[0]?.[1].table;
+		}
+		if (!table) throw new Error("runSQL requires at least one fileSet or an explicit table");
+		const result = await executor.execute({
+			sql: opts.sql,
+			params: opts.params ?? [],
+			fileKeys,
+			dataSource,
+			table,
+			signal: opts.signal
+		});
+		return {
+			rows: result.rows,
+			sql: result.sql,
+			objectKeys: uniqueKeys
+		};
+	}
+	async function query(ctx, state) {
+		const plan = buildLogicalPlan(state, { regex: true });
+		const table = ctx.table ?? plan.dataset;
+		const resolved = compileLogicalQueryPlan(plan, table);
+		return runSQL({
+			ctx: {
+				userId: ctx.userId,
+				siteId: ctx.siteId
+			},
+			table,
+			fileSets: { FILES: {
+				table,
+				partitions: resolved.partitions
+			} },
+			sql: resolved.sql,
+			params: resolved.params,
+			signal: ctx.signal
+		});
+	}
+	async function queryComparison(ctx, current, previous, filter) {
+		const adapter = createParquetResolverAdapter();
+		const currentPlan = buildLogicalPlan(current, adapter.capabilities);
+		const previousPlan = buildLogicalPlan(previous, adapter.capabilities);
+		if (currentPlan.dataset !== previousPlan.dataset) throw new Error(`queryComparison: current (${currentPlan.dataset}) and previous (${previousPlan.dataset}) must resolve to the same table`);
+		const table = ctx.table ?? currentPlan.dataset;
+		const comparison = resolveComparisonSQL(current, previous, {
+			adapter,
+			siteId: void 0
+		}, filter);
+		const totals = buildTotalsSql(current, {
+			adapter,
+			siteId: void 0
+		});
+		const fileSets = { FILES: {
+			table,
+			partitions: enumeratePartitions(currentPlan.dateRange.startDate < previousPlan.dateRange.startDate ? currentPlan.dateRange.startDate : previousPlan.dateRange.startDate, currentPlan.dateRange.endDate > previousPlan.dateRange.endDate ? currentPlan.dateRange.endDate : previousPlan.dateRange.endDate)
+		} };
+		const baseCtx = {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		};
+		const [main, count, totalsRow] = await Promise.all([
+			runSQL({
+				ctx: baseCtx,
+				table,
+				fileSets,
+				sql: comparison.sql,
+				params: comparison.params,
+				signal: ctx.signal
+			}),
+			runSQL({
+				ctx: baseCtx,
+				table,
+				fileSets,
+				sql: comparison.countSql,
+				params: comparison.countParams,
+				signal: ctx.signal
+			}),
+			runSQL({
+				ctx: baseCtx,
+				table,
+				fileSets,
+				sql: totals.sql,
+				params: totals.params,
+				signal: ctx.signal
+			})
+		]);
+		return {
+			rows: main.rows,
+			totalCount: Number(count.rows[0]?.total ?? 0),
+			totals: totalsRow.rows[0] ?? {}
+		};
+	}
+	async function queryExtras(ctx, state) {
+		const adapter = createParquetResolverAdapter();
+		const extras = buildExtrasQueries(state, {
+			adapter,
+			siteId: void 0
+		});
+		if (extras.length === 0) return [];
+		const plan = buildLogicalPlan(state, adapter.capabilities);
+		const table = ctx.table ?? plan.dataset;
+		const fileSets = { FILES: {
+			table,
+			partitions: enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate)
+		} };
+		const baseCtx = {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		};
+		const results = await Promise.all(extras.map((e) => runSQL({
+			ctx: baseCtx,
+			table,
+			fileSets,
+			sql: e.sql,
+			params: e.params,
+			signal: ctx.signal
+		})));
+		return extras.map((e, i) => ({
+			key: e.key,
+			rows: results[i].rows
+		}));
+	}
+	async function compactTiered(ctx, thresholds) {
+		return compactTieredImpl({
+			dataSource,
+			manifestStore,
+			codec
+		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
+	}
+	async function gcOrphans(ctx, graceMs) {
+		return gcOrphansImpl({
+			dataSource,
+			manifestStore
+		}, (ctx.now ?? defaultNow)(), graceMs, {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		});
+	}
+	async function purgeTenant(ctx) {
+		const prefix = tenantPrefix(ctx);
+		const keys = [];
+		const keyStream = dataSource.streamList ? dataSource.streamList(prefix) : async function* () {
+			for (const k of await dataSource.list(prefix)) yield k;
+		}();
+		for await (const key of keyStream) keys.push(key);
+		if (keys.length > 0) await dataSource.delete(keys);
+		const manifestResult = await manifestStore.purgeTenant({
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		});
+		return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			prefix,
+			objectsDeleted: keys.length,
+			entriesRemoved: manifestResult.entriesRemoved,
+			watermarksRemoved: manifestResult.watermarksRemoved,
+			syncStatesRemoved: manifestResult.syncStatesRemoved,
+			at: defaultNow()
+		};
+	}
+	async function purgeUrls(ctx, urls) {
+		const now = defaultNow();
+		const urlSet = new Set(urls);
+		let entriesRewritten = 0;
+		let rowsRemoved = 0;
+		let bytesAfter = 0;
+		if (urlSet.size === 0) return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			urlsRequested: 0,
+			entriesRewritten: 0,
+			rowsRemoved: 0,
+			bytesAfter: 0,
+			at: now
+		};
+		for (const table of URL_PURGE_TABLES) {
+			const entries = await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table
+			});
+			for (const entry of entries) await manifestStore.withLock({
+				userId: entry.userId,
+				siteId: entry.siteId,
+				table,
+				partition: entry.partition
+			}, async () => {
+				const rows = await codec.readRows({ table }, entry.objectKey, dataSource);
+				const kept = rows.filter((r) => typeof r.url !== "string" || !urlSet.has(r.url));
+				const removed = rows.length - kept.length;
+				if (removed === 0) return;
+				const searchType = entry.searchType;
+				const newKey = objectKey({
+					userId: entry.userId,
+					siteId: entry.siteId
+				}, table, entry.partition, now, searchType);
+				const { bytes, rowCount } = await codec.writeRows({ table }, kept, newKey, dataSource);
+				const newEntry = {
+					userId: entry.userId,
+					siteId: entry.siteId,
+					table,
+					partition: entry.partition,
+					objectKey: newKey,
+					rowCount,
+					bytes,
+					createdAt: now,
+					schemaVersion: entry.schemaVersion ?? currentSchemaVersion(table),
+					...entry.tier !== void 0 ? { tier: entry.tier } : {},
+					...searchType !== void 0 ? { searchType } : {}
+				};
+				await manifestStore.registerVersion(newEntry, [entry]);
+				entriesRewritten++;
+				rowsRemoved += removed;
+				bytesAfter += bytes;
+			});
+		}
+		return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			urlsRequested: urlSet.size,
+			entriesRewritten,
+			rowsRemoved,
+			bytesAfter,
+			at: now
+		};
+	}
+	return {
+		writeDay,
+		query,
+		queryComparison,
+		queryExtras,
+		runSQL,
+		compactTiered,
+		gcOrphans,
+		purgeTenant,
+		purgeUrls,
+		listLive: (filter) => manifestStore.listLive(filter),
+		listAll: (filter) => manifestStore.listAll(filter),
+		getWatermarks: (filter) => manifestStore.getWatermarks(filter),
+		getSyncStates: (filter) => manifestStore.getSyncStates(filter),
+		setSyncState: (scope, state, detail) => manifestStore.setSyncState(scope, state, detail),
+		readObject: (key) => dataSource.read(key)
+	};
+}
+export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, SCHEMAS, TABLE_METADATA, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };

package/dist/planner.mjs

@@ -1,2 +1,2 @@
-import { i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/compiler.mjs";
+import { i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.7.5",
+  "version": "0.8.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -71,21 +71,11 @@
       "import": "./dist/adapters/duckdb-node.mjs",
       "default": "./dist/adapters/duckdb-node.mjs"
     },
-    "./node-harness": {
-      "types": "./dist/adapters/node-harness.d.mts",
-      "import": "./dist/adapters/node-harness.mjs",
-      "default": "./dist/adapters/node-harness.mjs"
-    },
     "./filesystem": {
       "types": "./dist/adapters/filesystem.d.mts",
       "import": "./dist/adapters/filesystem.mjs",
       "default": "./dist/adapters/filesystem.mjs"
     },
-    "./http": {
-      "types": "./dist/adapters/http.d.mts",
-      "import": "./dist/adapters/http.mjs",
-      "default": "./dist/adapters/http.mjs"
-    },
     "./hyparquet": {
       "types": "./dist/adapters/hyparquet.d.mts",
       "import": "./dist/adapters/hyparquet.mjs",
@@ -135,16 +125,6 @@
       "types": "./dist/arrow-utils.d.mts",
       "import": "./dist/arrow-utils.mjs",
       "default": "./dist/arrow-utils.mjs"
-    },
-    "./inspection-sqlite-node": {
-      "types": "./dist/adapters/inspection-sqlite-node.d.mts",
-      "import": "./dist/adapters/inspection-sqlite-node.mjs",
-      "default": "./dist/adapters/inspection-sqlite-node.mjs"
-    },
-    "./inspection-sqlite-browser": {
-      "types": "./dist/adapters/inspection-sqlite-browser.d.mts",
-      "import": "./dist/adapters/inspection-sqlite-browser.mjs",
-      "default": "./dist/adapters/inspection-sqlite-browser.mjs"
     }
   },
   "main": "./dist/index.mjs",
@@ -157,38 +137,29 @@
   },
   "peerDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
-    "better-sqlite3": "^12.9.0",
     "hyparquet": "^1.25.6",
-    "hyparquet-writer": "^0.14.0",
-    "wa-sqlite": "^1.0.0"
+    "hyparquet-writer": "^0.14.0"
   },
   "peerDependenciesMeta": {
     "@duckdb/duckdb-wasm": {
       "optional": true
     },
-    "better-sqlite3": {
-      "optional": true
-    },
     "hyparquet": {
       "optional": true
     },
     "hyparquet-writer": {
       "optional": true
-    },
-    "wa-sqlite": {
-      "optional": true
     }
   },
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.7.5"
+    "gscdump": "0.8.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
-    "better-sqlite3": "^12.9.0",
     "hyparquet": "^1.25.6",
     "hyparquet-writer": "^0.14.0",
     "tsx": "^4.21.0",