@gscdump/engine 0.35.11 → 0.36.0

package/dist/_chunks/engine.mjs

@@ -432,7 +432,8 @@ function createStorageEngine(opts) {
 			await manifestStore.bumpWatermark({
 				userId: ctx.userId,
 				siteId: ctx.siteId,
-				table: ctx.table
+				table: ctx.table,
+				...searchType !== void 0 ? { searchType } : {}
 			}, date, now);
 		});
 	}
@@ -499,7 +500,8 @@ function createStorageEngine(opts) {
 			await manifestStore.bumpWatermark({
 				userId: ctx.userId,
 				siteId: ctx.siteId,
-				table: ctx.table
+				table: ctx.table,
+				...searchType !== void 0 ? { searchType } : {}
 			}, date, now);
 		});
 	}
@@ -613,11 +615,11 @@ function createStorageEngine(opts) {
 			for (const k of await dataSource.list(prefix)) yield k;
 		}();
 		for await (const key of keyStream) keys.push(key);
-		if (keys.length > 0) await dataSource.delete(keys);
 		const manifestResult = await manifestStore.purgeTenant({
 			userId: ctx.userId,
 			siteId: ctx.siteId
 		});
+		if (keys.length > 0) await dataSource.delete(keys);
 		return {
 			userId: ctx.userId,
 			siteId: ctx.siteId,

package/dist/_chunks/manifest-store-utils.mjs

@@ -3,7 +3,7 @@ function manifestEntryKey(entry) {
 	return entry.objectKey;
 }
 function watermarkKey(watermark) {
-	return `${watermark.userId}|${watermark.siteId ?? ""}|${watermark.table}`;
+	return `${watermark.userId}|${watermark.siteId ?? ""}|${watermark.table}|${inferSearchType(watermark)}`;
 }
 function syncStateKey(state) {
 	return `${state.userId}|${state.siteId ?? ""}|${state.table}|${state.date}|${inferSearchType(state)}`;
@@ -21,6 +21,7 @@ function matchesWatermarkFilter(watermark, filter, options = {}) {
 	if (!options.ignoreUserId && watermark.userId !== filter.userId) return false;
 	if (filter.siteId !== void 0 && watermark.siteId !== filter.siteId) return false;
 	if (filter.table !== void 0 && watermark.table !== filter.table) return false;
+	if (filter.searchType !== void 0 && inferSearchType(watermark) !== filter.searchType) return false;
 	return true;
 }
 function matchesSyncStateFilter(state, filter, options = {}) {

package/dist/_chunks/resolver.mjs

@@ -525,9 +525,10 @@ function joinAnd(parts) {
 function joinComma(parts) {
 	return sql.join(parts, sql`, `);
 }
-function orderByClause(state, prefix = "") {
+const ORDER_BY_HELPER_PREFIX = "__order_";
+function orderByClause(state, prefix = "", columnOverride) {
 	if (state.orderBy) {
-		const safeCol = state.orderBy.column.replace(/\W/g, "");
+		const safeCol = (columnOverride ?? state.orderBy.column).replace(/\W/g, "");
 		const safeDir = state.orderBy.dir.toUpperCase() === "ASC" ? "ASC" : "DESC";
 		return sql.raw(`ORDER BY ${prefix}${safeCol} ${safeDir}`);
 	}
@@ -670,12 +671,31 @@ function resolveToSQLOptimized(state, options) {
 			outerTotals.push(sql.raw("SUM(sum_position) OVER() / NULLIF(SUM(impressions) OVER(), 0) + 1 as totalPosition"));
 			break;
 	}
+	let orderByColumnOverride;
+	const orderColumn = state.orderBy?.column;
+	if (orderColumn && orderColumn !== "date" && !metrics.includes(orderColumn)) {
+		orderByColumnOverride = `${ORDER_BY_HELPER_PREFIX}${orderColumn}`;
+		switch (orderColumn) {
+			case "clicks":
+				outerSelect.push(sql.raw(`clicks as "${orderByColumnOverride}"`));
+				break;
+			case "impressions":
+				outerSelect.push(sql.raw(`impressions as "${orderByColumnOverride}"`));
+				break;
+			case "ctr":
+				outerSelect.push(sql.raw(`CAST(clicks AS REAL) / NULLIF(impressions, 0) as "${orderByColumnOverride}"`));
+				break;
+			case "position":
+				outerSelect.push(sql.raw(`sum_position / NULLIF(impressions, 0) + 1 as "${orderByColumnOverride}"`));
+				break;
+		}
+	}
 	outerSelect.push(sql.raw("COUNT(*) OVER() as totalCount"));
 	for (const totalExpr of outerTotals) outerSelect.push(totalExpr);
 	let cte = wherePredicates.length > 0 ? sql`SELECT ${joinComma(cteSelect)} FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT ${joinComma(cteSelect)} FROM ${table}`;
 	if (groupByExprs.length > 0) cte = sql`${cte} GROUP BY ${joinComma(groupByExprs)}`;
 	if (having.length > 0) cte = sql`${cte} HAVING ${joinAnd(having)}`;
-	return compileCollapsed(adapter, sql`WITH aggregated AS (${cte}) SELECT ${joinComma(outerSelect)} FROM aggregated ${orderByClause(state)} ${limitOffsetClause(state)}`);
+	return compileCollapsed(adapter, sql`WITH aggregated AS (${cte}) SELECT ${joinComma(outerSelect)} FROM aggregated ${orderByClause(state, "", orderByColumnOverride)} ${limitOffsetClause(state)}`);
 }
 function resolveToSQL(state, options) {
 	const { adapter } = options;
@@ -1106,6 +1126,7 @@ async function runOptimizedQuery(runSQL, ctx, state, dateRange, options = {}) {
 	return {
 		rows: optRes.rows.map((r) => {
 			const { totalCount: _tc, totalClicks: _tcl, totalImpressions: _ti, totalCtr: _tr, totalPosition: _tp, ...rest } = r;
+			for (const key of Object.keys(rest)) if (key.startsWith("__order_")) delete rest[key];
 			return rest;
 		}),
 		totalCount,

package/dist/_chunks/storage.d.mts

@@ -239,6 +239,11 @@ interface WatermarkScope {
   userId: string;
   siteId?: string;
   table: TableName;
+  /**
+   * GSC search-type this watermark covers. Omitted = `web` for legacy
+   * compatibility with pre-searchType watermarks.
+   */
+  searchType?: SearchType;
 }
 interface Watermark extends WatermarkScope {
   newestDateSynced: string;
@@ -249,6 +254,7 @@ interface WatermarkFilter {
   userId: string;
   siteId?: string;
   table?: TableName;
+  searchType?: SearchType;
 }
 type SyncStateKind = 'pending' | 'inflight' | 'done' | 'failed';
 interface SyncStateScope {
@@ -337,8 +343,9 @@ interface ManifestStore {
    * GDPR-grade tenant purge. Removes every manifest entry, watermark, and
    * sync-state record matching the filter. Does NOT touch the underlying
    * data-source bytes; callers (typically {@link StorageEngine.purgeTenant})
-   * must sweep the tenant prefix separately before invoking this so that
-   * mid-flight failures can't leave orphan parquet with no manifest record.
+   * remove manifest visibility first, then sweep tenant-prefix bytes so that
+   * mid-flight failures cannot leave live manifest entries pointing at deleted
+   * objects.
    *
    * On stores with CAS-backed sharding (R2 manifest) this may issue one
    * mutation per shard. On read-only stores (HTTP) this throws.

package/dist/adapters/filesystem.mjs

@@ -224,6 +224,7 @@ function createFilesystemManifestStore(opts) {
 					userId: scope.userId,
 					siteId: scope.siteId,
 					table: scope.table,
+					...scope.searchType !== void 0 ? { searchType: scope.searchType } : {},
 					newestDateSynced: date,
 					oldestDateSynced: date,
 					lastSyncAt: nowMs

package/dist/adapters/r2-manifest.mjs

@@ -124,12 +124,13 @@ function createR2ManifestStore(opts) {
 	async function mutateShard(siteId, table, mutate) {
 		return unwrapResult(await mutateShardResult(siteId, table, mutate), engineErrorToException);
 	}
-	async function listShards() {
+	async function listShards(siteId) {
 		const shards = [];
 		let cursor;
+		const prefix = siteId === void 0 ? `u_${userId}/manifest/` : `u_${userId}/manifest/${siteId}/`;
 		do {
 			const res = await bucket.list({
-				prefix: `u_${userId}/manifest/`,
+				prefix,
 				cursor,
 				limit: 1e3
 			});
@@ -149,7 +150,7 @@ function createR2ManifestStore(opts) {
 			siteId: filter.siteId,
 			table: filter.table
 		}];
-		return (await listShards()).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
+		return (await listShards(filter.siteId)).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
 	}
 	function assertScopedUser(got, op) {
 		if (got !== userId) throw new Error(`${op}: R2 manifest store is scoped to userId=${userId}, got ${got}`);
@@ -256,13 +257,15 @@ function createR2ManifestStore(opts) {
 			assertScopedUser(scope.userId, "bumpWatermark");
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
 			const ts = at ?? now();
+			const scopeSearchType = inferSearchType(scope);
 			await mutateShard(scope.siteId, scope.table, (snap) => {
-				const idx = snap.watermarks.findIndex((w) => w.userId === userId && w.siteId === scope.siteId && w.table === scope.table);
+				const idx = snap.watermarks.findIndex((w) => w.userId === userId && w.siteId === scope.siteId && w.table === scope.table && inferSearchType(w) === scopeSearchType);
 				if (idx === -1) {
 					snap.watermarks.push({
 						userId,
 						siteId: scope.siteId,
 						table: scope.table,
+						...scope.searchType !== void 0 ? { searchType: scope.searchType } : {},
 						newestDateSynced: date,
 						oldestDateSynced: date,
 						lastSyncAt: ts

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.35.11",
+  "version": "0.36.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.35.11",
-    "@gscdump/lakehouse": "0.35.11",
-    "gscdump": "0.35.11"
+    "@gscdump/contracts": "0.36.0",
+    "@gscdump/lakehouse": "0.36.0",
+    "gscdump": "0.36.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",