@gscdump/engine 0.4.0

package/dist/ingest.d.mts

@@ -0,0 +1,96 @@
+import { Row, TableName } from "gscdump/contracts";
+/**
+ * Canonical GSC API dimension order per table. Consumers hitting the raw
+ * `searchanalytics.query` endpoint must request dimensions in this order so
+ * that `transformGscRow` / `createRowAccumulator` can decode the resulting
+ * `keys[]` tuples. Storage-column names (e.g. `page` → `url`) are handled
+ * inside `transformGscRow` — this record stays in GSC-API vocabulary.
+ */
+declare const TABLE_DIMS: Record<TableName, string[]>;
+interface GscApiRow {
+  keys: string[];
+  clicks: number;
+  impressions: number;
+  /** Unused by ingest — the `sum_position` column encodes weighted position. */
+  ctr?: number;
+  position: number;
+}
+interface IngestOptions {
+  /**
+   * Canonical form of a query string, stored alongside `query` as
+   * `query_canonical`. Site-specific (e.g. synonym groups, stemming); if
+   * omitted, `query_canonical` is null. Applied to `keywords` +
+   * `page_keywords` tables only.
+   */
+  normalizeQuery?: (query: string) => string | null | undefined;
+}
+/**
+ * Strip a GSC URL to its pathname. Core analytics stores pages by path so
+ * queries don't carry origin-prefix filters.
+ */
+declare function toPath(gscUrl: string): string;
+/**
+ * Encode weighted average position as `sum_position`. The raw GSC position
+ * is 1-indexed; subtract 1 and weight by impressions so a downstream
+ * `SUM(sum_position) / SUM(impressions) + 1` recovers the true mean without
+ * ever materialising per-row position values.
+ */
+declare function toSumPosition(apiPosition: number, impressions: number): number;
+/**
+ * Map one GSC API row into `{ date, row }` for the given table, or null if
+ * the row has no keys (GSC occasionally emits empty-keys placeholders).
+ */
+declare function transformGscRow(table: TableName, apiRow: GscApiRow, options?: IngestOptions): {
+  date: string;
+  row: Row;
+} | null;
+interface RowAccumulator {
+  /**
+   * Push a batch of GSC API rows into the accumulator. Returns `false` if
+   * the batch pushed total row count past `maxRows`; subsequent pushes
+   * become no-ops until `drain()` is called.
+   */
+  push: (table: TableName, rows: readonly GscApiRow[]) => boolean;
+  /**
+   * Consume accumulated rows, grouped by `table → date → rows`. Resets
+   * internal state; subsequent pushes behave as on a fresh accumulator.
+   */
+  drain: () => Map<TableName, Map<string, Row[]>>;
+  /**
+   * Drain only buckets for dates strictly older than the most-recent date
+   * seen for each table. Requires `trackDateBoundary` to be enabled — without
+   * it, returns an empty map. GSC's date-as-dimension queries return rows
+   * sorted by date, so any date older than the latest seen is logically
+   * complete within the current job slice and safe to flush mid-job.
+   *
+   * Returned buckets are removed from internal state and `totalRows` is
+   * decremented accordingly. Latest-date buckets stay in place for the
+   * eventual `drain()` at job end.
+   */
+  drainCompleted: () => Map<TableName, Map<string, Row[]>>;
+  /** Total row count across all tables/dates since last drain. */
+  readonly totalRows: number;
+  /** Whether the accumulator has overflowed since last drain. */
+  readonly overflowed: boolean;
+}
+interface RowAccumulatorOptions extends IngestOptions {
+  /**
+   * Soft cap on total accumulated rows before `push` starts returning
+   * `false` and dropping rows. Defaults to 500_000 — matches the
+   * ~128 MB CF Workers isolate budget at ~200 bytes/row with headroom.
+   */
+  maxRows?: number;
+  /**
+   * Track the most-recent date seen per table so `drainCompleted()` can
+   * return older-date buckets mid-job. Off by default — callers that don't
+   * stream-flush pay zero overhead for the bookkeeping.
+   *
+   * Caller contract: only safe when GSC dimensions include `date` so the
+   * API returns rows in date-ascending order; without that ordering,
+   * "older than latest" doesn't mean "complete" and partial buckets would
+   * be flushed prematurely.
+   */
+  trackDateBoundary?: boolean;
+}
+declare function createRowAccumulator(options?: RowAccumulatorOptions): RowAccumulator;
+export { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, TABLE_DIMS, createRowAccumulator, toPath, toSumPosition, transformGscRow };

package/dist/ingest.mjs

@@ -0,0 +1,187 @@
+const TABLE_DIMS = {
+	pages: ["page", "date"],
+	keywords: ["query", "date"],
+	countries: ["country", "date"],
+	devices: ["device", "date"],
+	page_keywords: [
+		"page",
+		"query",
+		"date"
+	],
+	search_appearance: ["searchAppearance", "date"]
+};
+function toPath(gscUrl) {
+	try {
+		return new URL(gscUrl).pathname;
+	} catch {
+		return gscUrl;
+	}
+}
+function toSumPosition(apiPosition, impressions) {
+	return (apiPosition - 1) * Math.max(impressions, 1);
+}
+function transformGscRow(table, apiRow, options = {}) {
+	const keys = apiRow.keys;
+	if (!keys || keys.length === 0) return null;
+	const clicks = apiRow.clicks || 0;
+	const impressions = apiRow.impressions || 0;
+	const sum_position = toSumPosition(apiRow.position || 0, impressions);
+	if (table === "pages") {
+		const date = String(keys[1] ?? "");
+		return {
+			date,
+			row: {
+				url: toPath(String(keys[0] ?? "")),
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	if (table === "keywords") {
+		const query = String(keys[0] ?? "");
+		const date = String(keys[1] ?? "");
+		return {
+			date,
+			row: {
+				query,
+				query_canonical: options.normalizeQuery?.(query) ?? null,
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	if (table === "countries") {
+		const date = String(keys[1] ?? "");
+		return {
+			date,
+			row: {
+				country: String(keys[0] ?? ""),
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	if (table === "devices") {
+		const date = String(keys[1] ?? "");
+		return {
+			date,
+			row: {
+				device: String(keys[0] ?? ""),
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	if (table === "search_appearance") {
+		const date = String(keys[1] ?? "");
+		return {
+			date,
+			row: {
+				searchAppearance: String(keys[0] ?? ""),
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	const query = String(keys[1] ?? "");
+	const date = String(keys[2] ?? "");
+	const query_canonical = options.normalizeQuery?.(query) ?? null;
+	return {
+		date,
+		row: {
+			url: toPath(String(keys[0] ?? "")),
+			query,
+			query_canonical,
+			date,
+			clicks,
+			impressions,
+			sum_position
+		}
+	};
+}
+const DEFAULT_MAX_ROWS = 5e5;
+function createRowAccumulator(options = {}) {
+	const maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
+	const trackDateBoundary = options.trackDateBoundary === true;
+	let buckets = /* @__PURE__ */ new Map();
+	const latestDate = /* @__PURE__ */ new Map();
+	let total = 0;
+	let overflowed = false;
+	function bucketFor(table, date) {
+		let byDate = buckets.get(table);
+		if (!byDate) {
+			byDate = /* @__PURE__ */ new Map();
+			buckets.set(table, byDate);
+		}
+		let rows = byDate.get(date);
+		if (!rows) {
+			rows = [];
+			byDate.set(date, rows);
+		}
+		return rows;
+	}
+	return {
+		get totalRows() {
+			return total;
+		},
+		get overflowed() {
+			return overflowed;
+		},
+		push(table, rows) {
+			if (overflowed) return false;
+			for (const r of rows) {
+				const t = transformGscRow(table, r, options);
+				if (!t || !t.date) continue;
+				bucketFor(table, t.date).push(t.row);
+				total++;
+				if (trackDateBoundary) {
+					const prev = latestDate.get(table);
+					if (!prev || t.date > prev) latestDate.set(table, t.date);
+				}
+				if (total > maxRows) {
+					overflowed = true;
+					return false;
+				}
+			}
+			return true;
+		},
+		drain() {
+			const out = buckets;
+			buckets = /* @__PURE__ */ new Map();
+			latestDate.clear();
+			total = 0;
+			overflowed = false;
+			return out;
+		},
+		drainCompleted() {
+			const out = /* @__PURE__ */ new Map();
+			if (!trackDateBoundary) return out;
+			for (const [table, byDate] of buckets) {
+				const latest = latestDate.get(table);
+				if (!latest) continue;
+				let outBy;
+				for (const [date, dateRows] of byDate) if (date < latest) {
+					if (!outBy) {
+						outBy = /* @__PURE__ */ new Map();
+						out.set(table, outBy);
+					}
+					outBy.set(date, dateRows);
+					total -= dateRows.length;
+				}
+				if (outBy) for (const date of outBy.keys()) byDate.delete(date);
+			}
+			return out;
+		}
+	};
+}
+export { TABLE_DIMS, createRowAccumulator, toPath, toSumPosition, transformGscRow };

package/dist/planner.d.mts

@@ -0,0 +1,16 @@
+import { LogicalQueryPlan } from "gscdump/query/plan";
+import { TableName } from "gscdump/contracts";
+import { BuilderState } from "gscdump/query";
+declare function enumeratePartitions(startDate: string, endDate: string): string[];
+interface ResolvedQuery {
+  sql: string;
+  params: unknown[];
+  partitions: string[];
+  table: TableName;
+  filesPlaceholder: string;
+}
+declare const FILES_PLACEHOLDER = "{{FILES}}";
+declare function compileLogicalQueryPlan(plan: LogicalQueryPlan, table?: TableName): ResolvedQuery;
+declare function resolveToSQL(state: BuilderState, table?: TableName): ResolvedQuery;
+declare function substituteNamedFiles(sql: string, sets: Record<string, string[]>): string;
+export { FILES_PLACEHOLDER, type ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/dist/planner.mjs

@@ -0,0 +1,321 @@
+import { MS_PER_DAY, toIsoDate } from "gscdump";
+import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
+import { buildLogicalPlan } from "gscdump/query/plan";
+function metricCols() {
+	return {
+		clicks: integer("clicks").notNull(),
+		impressions: integer("impressions").notNull(),
+		sum_position: doublePrecision("sum_position").notNull()
+	};
+}
+const dateCol = () => date("date").notNull();
+const drizzleSchema = {
+	pages: pgTable("pages", {
+		url: varchar("url").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	keywords: pgTable("keywords", {
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	countries: pgTable("countries", {
+		country: varchar("country").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	devices: pgTable("devices", {
+		device: varchar("device").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	page_keywords: pgTable("page_keywords", {
+		url: varchar("url").notNull(),
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	search_appearance: pgTable("search_appearance", {
+		searchAppearance: varchar("searchAppearance").notNull(),
+		date: dateCol(),
+		...metricCols()
+	})
+};
+const TABLE_METADATA = {
+	pages: {
+		sortKey: ["date", "url"],
+		version: 1
+	},
+	keywords: {
+		sortKey: ["date", "query"],
+		version: 2
+	},
+	countries: {
+		sortKey: ["date", "country"],
+		version: 1
+	},
+	devices: {
+		sortKey: ["date", "device"],
+		version: 1
+	},
+	page_keywords: {
+		sortKey: [
+			"date",
+			"url",
+			"query"
+		],
+		version: 2
+	},
+	search_appearance: {
+		sortKey: ["date", "searchAppearance"],
+		version: 1
+	}
+};
+function pgSqlTypeToColumnType(sqlType) {
+	const t = sqlType.toLowerCase();
+	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
+	if (t === "date" || t.startsWith("timestamp")) return "DATE";
+	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
+	if (t === "bigint" || t === "int8") return "BIGINT";
+	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
+	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
+}
+function tableSchemaFrom(tableName) {
+	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
+		name: col.name,
+		type: pgSqlTypeToColumnType(col.getSQLType()),
+		nullable: !col.notNull
+	}));
+	const meta = TABLE_METADATA[tableName];
+	return {
+		name: tableName,
+		columns,
+		sortKey: meta.sortKey,
+		version: meta.version
+	};
+}
+Object.fromEntries([
+	"pages",
+	"keywords",
+	"countries",
+	"devices",
+	"page_keywords",
+	"search_appearance"
+].map((t) => [t, tableSchemaFrom(t)]));
+function dimensionToColumn(dim, _table) {
+	if (dim === "page") return "url";
+	if (dim === "queryCanonical") return "query_canonical";
+	return dim;
+}
+function dayPartition(date) {
+	return `daily/${date}`;
+}
+function monthPartition(month) {
+	return `monthly/${month}`;
+}
+function weekPartition(mondayIsoDate) {
+	return `weekly/${mondayIsoDate}`;
+}
+function quarterPartition(quarter) {
+	return `quarterly/${quarter}`;
+}
+function mondayOfWeek(isoDate) {
+	const ms = Date.parse(`${isoDate}T00:00:00Z`);
+	const dow = new Date(ms).getUTCDay();
+	const offset = dow === 0 ? -6 : 1 - dow;
+	return toIsoDate(new Date(ms + offset * MS_PER_DAY));
+}
+function quarterOfMonth(month) {
+	const [y, m] = month.split("-").map(Number);
+	return `${y}-Q${Math.floor((m - 1) / 3) + 1}`;
+}
+const DEFAULT_THRESHOLDS = {
+	raw: 7,
+	d7: 30,
+	d30: 90
+};
+DEFAULT_THRESHOLDS.raw, DEFAULT_THRESHOLDS.d7, DEFAULT_THRESHOLDS.d30;
+function enumeratePartitions(startDate, endDate) {
+	const out = [];
+	const [sy, sm, sd] = startDate.split("-").map(Number);
+	const [ey, em, ed] = endDate.split("-").map(Number);
+	const start = Date.UTC(sy, sm - 1, sd);
+	const end = Date.UTC(ey, em - 1, ed);
+	if (end < start) return out;
+	const seenWeeks = /* @__PURE__ */ new Set();
+	const seenMonths = /* @__PURE__ */ new Set();
+	const seenQuarters = /* @__PURE__ */ new Set();
+	for (let t = start; t <= end; t += 864e5) {
+		const d = new Date(t);
+		const y = d.getUTCFullYear();
+		const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+		const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
+		const isoMonth = `${y}-${m}`;
+		out.push(dayPartition(isoDay));
+		const monday = mondayOfWeek(isoDay);
+		if (!seenWeeks.has(monday)) {
+			seenWeeks.add(monday);
+			out.push(weekPartition(monday));
+		}
+		if (!seenMonths.has(isoMonth)) {
+			seenMonths.add(isoMonth);
+			out.push(monthPartition(isoMonth));
+		}
+		const quarter = quarterOfMonth(isoMonth);
+		if (!seenQuarters.has(quarter)) {
+			seenQuarters.add(quarter);
+			out.push(quarterPartition(quarter));
+		}
+	}
+	return out;
+}
+function escapeLike(value) {
+	return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
+}
+const METRIC_EXPR = {
+	clicks: "CAST(SUM(clicks) AS DOUBLE)",
+	impressions: "CAST(SUM(impressions) AS DOUBLE)",
+	ctr: "CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0)",
+	position: "SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1"
+};
+function topLevelPagePredicateSql(pathExpr) {
+	return `LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
+}
+const FILES_PLACEHOLDER = "{{FILES}}";
+function buildDimensionWhere(filters, table) {
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const column = dimensionToColumn(filter.dimension, table);
+		switch (filter.operator) {
+			case "equals":
+				clauses.push(`${column} = ?`);
+				params.push(filter.expression);
+				break;
+			case "notEquals":
+				clauses.push(`${column} != ?`);
+				params.push(filter.expression);
+				break;
+			case "contains":
+				clauses.push(`${column} LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "notContains":
+				clauses.push(`${column} NOT LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "includingRegex":
+				clauses.push(`regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+			case "excludingRegex":
+				clauses.push(`NOT regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.join(" AND "),
+		params
+	};
+}
+function buildTopLevelWhere(plan, table) {
+	if (!plan.specialFilters.topLevel) return "";
+	return topLevelPagePredicateSql(dimensionToColumn("page", table));
+}
+function buildHaving(filters) {
+	if (filters.length === 0) return {
+		clause: "",
+		params: []
+	};
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const expr = METRIC_EXPR[filter.metric];
+		switch (filter.operator) {
+			case "metricGte":
+				clauses.push(`${expr} >= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricGt":
+				clauses.push(`${expr} > ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLte":
+				clauses.push(`${expr} <= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLt":
+				clauses.push(`${expr} < ?`);
+				params.push(filter.expression);
+				break;
+			case "metricBetween":
+				clauses.push(`${expr} >= ? AND ${expr} <= ?`);
+				params.push(filter.expression, filter.expression2 ?? filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.length > 0 ? `HAVING ${clauses.join(" AND ")}` : "",
+		params
+	};
+}
+function compileLogicalQueryPlan(plan, table = plan.dataset) {
+	const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
+	const metricSelects = plan.metrics.map((metric) => `${METRIC_EXPR[metric]} AS ${metric}`);
+	const dimSelects = plan.groupByDimensions.map((dimension) => {
+		const column = dimensionToColumn(dimension, table);
+		return column !== dimension ? `${column} AS ${dimension}` : dimension;
+	});
+	const whereClauses = ["date >= ?", "date <= ?"];
+	const whereParams = [plan.dateRange.startDate, plan.dateRange.endDate];
+	const dimWhere = buildDimensionWhere(plan.dimensionFilters, table);
+	if (dimWhere.clause) {
+		whereClauses.push(dimWhere.clause);
+		whereParams.push(...dimWhere.params);
+	}
+	const topLevelClause = buildTopLevelWhere(plan, table);
+	if (topLevelClause) whereClauses.push(topLevelClause);
+	const having = buildHaving(plan.metricFilters);
+	const groupByCols = [...plan.groupByDimensions.map((dimension) => dimensionToColumn(dimension, table)), ...plan.hasDate ? ["date"] : []];
+	const groupBy = groupByCols.length > 0 ? `GROUP BY ${groupByCols.join(", ")}` : "";
+	const orderBy = plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC";
+	const limit = `LIMIT ${plan.rowLimit ?? 1e3}`;
+	const offset = plan.startRow ? `OFFSET ${plan.startRow}` : "";
+	return {
+		sql: [
+			`SELECT ${[
+				...dimSelects,
+				...plan.hasDate ? ["date"] : [],
+				...metricSelects
+			].join(", ")}`,
+			`FROM read_parquet(${FILES_PLACEHOLDER}, union_by_name = true)`,
+			`WHERE ${whereClauses.join(" AND ")}`,
+			groupBy,
+			having.clause,
+			orderBy,
+			limit,
+			offset
+		].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(),
+		params: [...whereParams, ...having.params],
+		partitions,
+		table,
+		filesPlaceholder: FILES_PLACEHOLDER
+	};
+}
+function resolveToSQL(state, table) {
+	const plan = buildLogicalPlan(state, { regex: true });
+	return compileLogicalQueryPlan(plan, table ?? plan.dataset);
+}
+function fileList(keys) {
+	return keys.length === 0 ? "[]" : `[${keys.map((key) => `'${key.replace(/'/g, "''")}'`).join(", ")}]`;
+}
+function substituteNamedFiles(sql, sets) {
+	let out = sql;
+	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
+	return out;
+}
+export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };