@gscdump/engine 0.4.0

package/dist/rollups.mjs

@@ -0,0 +1,553 @@
+import { MS_PER_DAY } from "gscdump";
+import { parquetWriteBuffer } from "hyparquet-writer";
+import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
+function metricCols() {
+	return {
+		clicks: integer("clicks").notNull(),
+		impressions: integer("impressions").notNull(),
+		sum_position: doublePrecision("sum_position").notNull()
+	};
+}
+const dateCol = () => date("date").notNull();
+const drizzleSchema = {
+	pages: pgTable("pages", {
+		url: varchar("url").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	keywords: pgTable("keywords", {
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	countries: pgTable("countries", {
+		country: varchar("country").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	devices: pgTable("devices", {
+		device: varchar("device").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	page_keywords: pgTable("page_keywords", {
+		url: varchar("url").notNull(),
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	search_appearance: pgTable("search_appearance", {
+		searchAppearance: varchar("searchAppearance").notNull(),
+		date: dateCol(),
+		...metricCols()
+	})
+};
+const TABLE_METADATA = {
+	pages: {
+		sortKey: ["date", "url"],
+		version: 1
+	},
+	keywords: {
+		sortKey: ["date", "query"],
+		version: 2
+	},
+	countries: {
+		sortKey: ["date", "country"],
+		version: 1
+	},
+	devices: {
+		sortKey: ["date", "device"],
+		version: 1
+	},
+	page_keywords: {
+		sortKey: [
+			"date",
+			"url",
+			"query"
+		],
+		version: 2
+	},
+	search_appearance: {
+		sortKey: ["date", "searchAppearance"],
+		version: 1
+	}
+};
+function pgSqlTypeToColumnType(sqlType) {
+	const t = sqlType.toLowerCase();
+	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
+	if (t === "date" || t.startsWith("timestamp")) return "DATE";
+	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
+	if (t === "bigint" || t === "int8") return "BIGINT";
+	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
+	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
+}
+function tableSchemaFrom(tableName) {
+	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
+		name: col.name,
+		type: pgSqlTypeToColumnType(col.getSQLType()),
+		nullable: !col.notNull
+	}));
+	const meta = TABLE_METADATA[tableName];
+	return {
+		name: tableName,
+		columns,
+		sortKey: meta.sortKey,
+		version: meta.version
+	};
+}
+Object.fromEntries([
+	"pages",
+	"keywords",
+	"countries",
+	"devices",
+	"page_keywords",
+	"search_appearance"
+].map((t) => [t, tableSchemaFrom(t)]));
+const ROW_GROUP_SIZE = 25e3;
+function basicTypeFor(colType) {
+	if (colType === "VARCHAR" || colType === "DATE") return "STRING";
+	if (colType === "BIGINT") return "INT64";
+	if (colType === "INTEGER") return "INT32";
+	if (colType === "DOUBLE") return "DOUBLE";
+	throw new Error(`unsupported column type for parquet encoding: ${colType}`);
+}
+function coerceValue(value, type) {
+	if (value === null || value === void 0) return null;
+	if (type === "STRING") return typeof value === "string" ? value : String(value);
+	if (type === "INT32") {
+		const n = typeof value === "number" ? value : Number(value);
+		if (!Number.isFinite(n)) throw new Error(`non-finite number for INT32: ${String(value)}`);
+		return Math.trunc(n);
+	}
+	if (type === "INT64") {
+		if (typeof value === "bigint") return value;
+		const n = typeof value === "number" ? value : Number(value);
+		if (!Number.isFinite(n)) throw new Error(`non-finite number for INT64: ${String(value)}`);
+		return BigInt(Math.trunc(n));
+	}
+	if (type === "DOUBLE") {
+		const n = typeof value === "number" ? value : Number(value);
+		if (!Number.isFinite(n)) throw new Error(`non-finite number for DOUBLE: ${String(value)}`);
+		return n;
+	}
+	return value;
+}
+function compareValues(a, b) {
+	if (a === b) return 0;
+	if (a === null || a === void 0) return -1;
+	if (b === null || b === void 0) return 1;
+	if (typeof a === "number" && typeof b === "number") return a - b;
+	return String(a) < String(b) ? -1 : 1;
+}
+function encodeRowsToParquetFlex(rows, opts) {
+	const { columns, sortKey = [], rowGroupSize = ROW_GROUP_SIZE } = opts;
+	const sorted = sortKey.length === 0 || rows.length <= 1 ? rows : [...rows].sort((a, b) => {
+		for (const col of sortKey) {
+			const cmp = compareValues(a[col], b[col]);
+			if (cmp !== 0) return cmp;
+		}
+		return 0;
+	});
+	const buffer = parquetWriteBuffer({
+		columnData: columns.map((col) => {
+			const type = basicTypeFor(col.type);
+			const data = sorted.map((r) => coerceValue(r[col.name], type));
+			return {
+				name: col.name,
+				data,
+				type,
+				nullable: col.nullable,
+				columnIndex: true
+			};
+		}),
+		rowGroupSize
+	});
+	return new Uint8Array(buffer);
+}
+function hashUrl(url) {
+	let hi = 2166136261;
+	let lo = 3421674724;
+	for (let i = 0; i < url.length; i++) {
+		const c = url.charCodeAt(i);
+		lo ^= c;
+		const loMul = Math.imul(lo, 435) >>> 0;
+		const carry = Math.floor(lo * 435 / 4294967296);
+		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
+		lo = loMul;
+		hi = hiMul;
+	}
+	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
+}
+function indexingMetadataIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/indexing/index.json` : `u_${ctx.userId}/entities/indexing/index.json`;
+}
+function createIndexingMetadataStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function readIndex(key) {
+		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => ({
+			version: 1,
+			records: {}
+		}));
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			const key = indexingMetadataIndexKey(ctx);
+			const index = await readIndex(key);
+			for (const r of records) index.records[hash(r.url)] = r;
+			await ds.write(key, new TextEncoder().encode(JSON.stringify(index)));
+		},
+		async loadIndex(ctx) {
+			return readIndex(indexingMetadataIndexKey(ctx));
+		},
+		async getLatest(ctx, url) {
+			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
+		}
+	};
+}
+function rollupPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
+}
+function rollupKey(ctx, id, builtAt) {
+	return `${rollupPrefix(ctx)}/${id}__v${builtAt}.json`;
+}
+function rollupParquetKey(ctx, id, builtAt) {
+	return `${rollupPrefix(ctx)}/${id}__v${builtAt}.parquet`;
+}
+async function rebuildRollups(opts) {
+	const now = opts.now ?? (() => Date.now());
+	const results = [];
+	for (const def of opts.defs) {
+		const builtAt = now();
+		const payload = await def.build({
+			engine: opts.engine,
+			ctx: opts.ctx,
+			dataSource: opts.dataSource,
+			builtAt
+		});
+		if (def.format === "parquet") {
+			if (!def.parquetColumns || def.parquetColumns.length === 0) throw new Error(`rollup '${def.id}' declared format='parquet' without parquetColumns`);
+			const rows = payload ?? [];
+			const parquetBytes = encodeRowsToParquetFlex(rows, {
+				columns: def.parquetColumns,
+				sortKey: def.parquetSortKey
+			});
+			const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt);
+			await opts.dataSource.write(parquetKey, parquetBytes);
+			const pointer = {
+				parquetKey,
+				rowCount: rows.length
+			};
+			const envelope = {
+				version: 1,
+				id: def.id,
+				builtAt,
+				windowDays: def.windowDays,
+				payload: pointer
+			};
+			const envelopeBytes = new TextEncoder().encode(JSON.stringify(envelope));
+			const key = rollupKey(opts.ctx, def.id, builtAt);
+			await opts.dataSource.write(key, envelopeBytes);
+			results.push({
+				id: def.id,
+				objectKey: key,
+				parquetKey,
+				bytes: envelopeBytes.byteLength,
+				parquetBytes: parquetBytes.byteLength,
+				builtAt
+			});
+			continue;
+		}
+		const envelope = {
+			version: 1,
+			id: def.id,
+			builtAt,
+			windowDays: def.windowDays,
+			payload
+		};
+		const json = JSON.stringify(envelope);
+		const bytes = new TextEncoder().encode(json);
+		const key = rollupKey(opts.ctx, def.id, builtAt);
+		await opts.dataSource.write(key, bytes);
+		results.push({
+			id: def.id,
+			objectKey: key,
+			bytes: bytes.byteLength,
+			builtAt
+		});
+	}
+	return results;
+}
+function utcDateMinusDays(at, days) {
+	const d = new Date(at - days * MS_PER_DAY);
+	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
+}
+const dailyTotalsRollup = {
+	id: "daily_totals",
+	windowDays: null,
+	async build({ engine, ctx }) {
+		const pages = await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { FILES: { table: "pages" } },
+			sql: `
+        SELECT
+          date,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        GROUP BY date
+        ORDER BY date
+      `
+		});
+		const keywords = await engine.runSQL({
+			ctx,
+			table: "keywords",
+			fileSets: { FILES: { table: "keywords" } },
+			sql: `
+        SELECT
+          date,
+          SUM(impressions)::BIGINT AS impressions
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        GROUP BY date
+      `
+		});
+		const keywordImpressionsByDate = /* @__PURE__ */ new Map();
+		for (const r of keywords.rows) keywordImpressionsByDate.set(String(r.date), BigInt(r.impressions));
+		return pages.rows.map((r) => {
+			const totalImpressions = BigInt(r.impressions);
+			const queryImpressions = keywordImpressionsByDate.get(String(r.date)) ?? BigInt(0);
+			const anonymized = totalImpressions === BigInt(0) ? 0 : 1 - Number(queryImpressions) / Number(totalImpressions);
+			return {
+				date: r.date,
+				clicks: Number(r.clicks),
+				impressions: Number(r.impressions),
+				sum_position: Number(r.sum_position),
+				anonymizedImpressionsPct: Math.max(0, Math.min(1, anonymized))
+			};
+		});
+	}
+};
+const weeklyTotalsRollup = {
+	id: "weekly_totals",
+	windowDays: null,
+	async build({ engine, ctx }) {
+		return (await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { FILES: { table: "pages" } },
+			sql: `
+        SELECT
+          strftime(date_trunc('week', date::DATE), '%Y-%m-%d') AS week,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        GROUP BY 1
+        ORDER BY 1
+      `
+		})).rows.map((r) => ({
+			week: r.week,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topPages28dRollup = {
+	id: "top_pages_28d",
+	windowDays: 28,
+	async build({ engine, ctx, builtAt }) {
+		const cutoff = utcDateMinusDays(builtAt, 28);
+		return (await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { FILES: { table: "pages" } },
+			sql: `
+        SELECT
+          url,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY url
+        ORDER BY clicks DESC
+        LIMIT 1000
+      `
+		})).rows.map((r) => ({
+			url: r.url,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topCountries28dRollup = {
+	id: "top_countries_28d",
+	windowDays: 28,
+	async build({ engine, ctx, builtAt }) {
+		const cutoff = utcDateMinusDays(builtAt, 28);
+		return (await engine.runSQL({
+			ctx,
+			table: "countries",
+			fileSets: { FILES: { table: "countries" } },
+			sql: `
+        SELECT
+          country,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY country
+        ORDER BY clicks DESC
+        LIMIT 250
+      `
+		})).rows.map((r) => ({
+			country: r.country,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topKeywords28dRollup = {
+	id: "top_keywords_28d",
+	windowDays: 28,
+	async build({ engine, ctx, builtAt }) {
+		const cutoff = utcDateMinusDays(builtAt, 28);
+		return (await engine.runSQL({
+			ctx,
+			table: "keywords",
+			fileSets: { FILES: { table: "keywords" } },
+			sql: `
+        SELECT
+          query,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY query
+        ORDER BY clicks DESC
+        LIMIT 1000
+      `
+		})).rows.map((r) => ({
+			query: r.query,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topKeywords28dParquetRollup = {
+	id: "top_keywords_28d_parquet",
+	windowDays: 28,
+	format: "parquet",
+	parquetColumns: [
+		{
+			name: "query",
+			type: "VARCHAR",
+			nullable: false
+		},
+		{
+			name: "clicks",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "impressions",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "sum_position",
+			type: "DOUBLE",
+			nullable: false
+		}
+	],
+	parquetSortKey: ["clicks"],
+	async build({ engine, ctx, builtAt }) {
+		const cutoff = utcDateMinusDays(builtAt, 28);
+		return (await engine.runSQL({
+			ctx,
+			table: "keywords",
+			fileSets: { FILES: { table: "keywords" } },
+			sql: `
+        SELECT
+          query,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY query
+        ORDER BY clicks DESC
+        LIMIT 1000
+      `
+		})).rows.map((r) => ({
+			query: String(r.query),
+			clicks: BigInt(r.clicks),
+			impressions: BigInt(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const indexingMetadataRollup = {
+	id: "indexing_metadata",
+	windowDays: null,
+	async build({ dataSource, ctx }) {
+		const index = await createIndexingMetadataStore({ dataSource }).loadIndex(ctx);
+		const records = Object.values(index.records);
+		const updatesByDay = /* @__PURE__ */ new Map();
+		const removesByDay = /* @__PURE__ */ new Map();
+		let totalUpdates = 0;
+		let totalRemoves = 0;
+		let latestUpdate;
+		let latestRemove;
+		for (const r of records) {
+			if (r.latestUpdateAt) {
+				totalUpdates++;
+				const day = r.latestUpdateAt.slice(0, 10);
+				updatesByDay.set(day, (updatesByDay.get(day) ?? 0) + 1);
+				if (!latestUpdate || r.latestUpdateAt > latestUpdate) latestUpdate = r.latestUpdateAt;
+			}
+			if (r.latestRemoveAt) {
+				totalRemoves++;
+				const day = r.latestRemoveAt.slice(0, 10);
+				removesByDay.set(day, (removesByDay.get(day) ?? 0) + 1);
+				if (!latestRemove || r.latestRemoveAt > latestRemove) latestRemove = r.latestRemoveAt;
+			}
+		}
+		const days = new Set([...updatesByDay.keys(), ...removesByDay.keys()]);
+		const perDay = Array.from(days).sort().map((day) => ({
+			day,
+			updates: updatesByDay.get(day) ?? 0,
+			removes: removesByDay.get(day) ?? 0
+		}));
+		return {
+			totals: {
+				urls: records.length,
+				updates: totalUpdates,
+				removes: totalRemoves,
+				latestUpdateAt: latestUpdate ?? null,
+				latestRemoveAt: latestRemove ?? null
+			},
+			days: perDay
+		};
+	}
+};
+const DEFAULT_ROLLUPS = [
+	dailyTotalsRollup,
+	weeklyTotalsRollup,
+	topPages28dRollup,
+	topKeywords28dRollup,
+	topCountries28dRollup,
+	indexingMetadataRollup
+];
+export { DEFAULT_ROLLUPS, dailyTotalsRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };