@gscdump/engine 1.3.2 → 1.4.1

package/dist/resolver/filter-utils.mjs

@@ -0,0 +1,59 @@
+import { normalizeUrl } from "gscdump/normalize";
+function collectInternalFilters(filter) {
+	if (!filter || !("_filters" in filter)) return [];
+	const flat = filter._filters;
+	const nested = filter._nestedGroups?.flatMap((group) => collectInternalFilters(group)) ?? [];
+	return [...flat, ...nested];
+}
+function getInternalFilters(filter) {
+	return collectInternalFilters(filter);
+}
+function getDimensionFilters(filter, isMetricDimension) {
+	return collectInternalFilters(filter).filter((f) => f.dimension !== "date" && !isMetricDimension(f.dimension) && f.operator !== "topLevel" && !f.operator.startsWith("metric"));
+}
+function getFilterDimensions(filter, isMetricDimension) {
+	return getDimensionFilters(filter, isMetricDimension).map((f) => f.dimension);
+}
+function metricValue(row, metric) {
+	const value = row[metric];
+	if (typeof value === "number") return value;
+	if (typeof value === "bigint") return Number(value);
+	if (value == null) return 0;
+	return Number(value);
+}
+function dimensionValue(row, dimension) {
+	const value = row[dimension];
+	return value == null ? "" : String(value);
+}
+function matchesDimensionFilter(row, filter) {
+	const raw = dimensionValue(row, filter.dimension);
+	const value = filter.dimension === "page" ? normalizeUrl(raw) : raw;
+	switch (filter.operator) {
+		case "equals": return value === filter.expression;
+		case "notEquals": return value !== filter.expression;
+		case "contains": return raw.includes(filter.expression);
+		case "notContains": return !raw.includes(filter.expression);
+		case "includingRegex": return new RegExp(filter.expression).test(raw);
+		case "excludingRegex": return !new RegExp(filter.expression).test(raw);
+		default: return true;
+	}
+}
+function matchesMetricFilter(row, filter) {
+	const value = metricValue(row, filter.dimension);
+	const target = Number(filter.expression);
+	switch (filter.operator) {
+		case "metricGte": return value >= target;
+		case "metricGt": return value > target;
+		case "metricLte": return value <= target;
+		case "metricLt": return value < target;
+		case "metricBetween": return value >= target && value <= Number(filter.expression2);
+		default: return true;
+	}
+}
+function matchesTopLevelPage(row) {
+	const path = normalizeUrl(dimensionValue(row, "page"));
+	let slashes = 0;
+	for (let index = 0; index < path.length; index++) if (path.charCodeAt(index) === 47 && ++slashes > 1) return false;
+	return true;
+}
+export { dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, metricValue };

package/dist/resolver/fragments.d.mts

@@ -0,0 +1,60 @@
+import { LOGICAL_DATASETS } from "./datasets.mjs";
+import { LogicalDataset } from "gscdump/query/plan";
+import { SQL } from "drizzle-orm";
+import { Dimension, InternalFilter, Metric } from "gscdump/query";
+interface SqlFragmentsConfig<TableKey extends string> {
+  schema: Record<TableKey, unknown>;
+  datasetToTableKey: Record<keyof typeof LOGICAL_DATASETS, TableKey>;
+  metricCast: 'REAL' | 'DOUBLE';
+  regexPredicate: (expr: SQL, pattern: string, negate: boolean) => SQL;
+  tableLabel: string;
+  includeSiteId: boolean;
+  includeSearchType?: boolean;
+  urlToPathExpr?: (col: string) => string;
+  /**
+   * Override the FROM-clause table reference. Default emits the bound drizzle
+   * table (e.g. `"pages"`). Parquet/R2 adapter overrides this to emit
+   * `read_parquet({{FILES}}, ...) AS "${tk}"` so the runSQL pipeline can swap
+   * in an object-key list while column refs (`"pages"."url"`) still resolve
+   * against the alias.
+   */
+  tableRef?: (tableKey: TableKey) => SQL;
+  /**
+   * Where `queryCanonical` comes from when compiling fact-table reads.
+   *
+   * - `queryDim` (default): LEFT JOIN a query dimension relation and derive
+   *   `COALESCE(query_dim.query_canonical, fact.query)`.
+   * - `column`: read a `query_canonical` column from the primary relation. This
+   *   is reserved for derived canonical rollup relations, not fact tables.
+   */
+  queryCanonicalSource?: 'queryDim' | 'column';
+  /** Relation used for the query-dimension join when `queryCanonicalSource=queryDim`. */
+  queryDimTableRef?: () => SQL;
+  /** When true, also joins query_dim on site_id (SQLite/D1 dimension table). */
+  queryDimSiteScoped?: boolean;
+}
+interface SqlFragments<TableKey extends string> {
+  METRIC_NAMES: Metric[];
+  DIM_COLUMN_MAP: Record<TableKey, Record<string, string>>;
+  isMetricDimension: (dim: string) => dim is Metric;
+  tableKeyForDataset: (dataset: LogicalDataset) => TableKey;
+  dimColumn: (dim: Dimension, table: TableKey) => string;
+  inferTable: (dimensions: Dimension[], filterDims?: Dimension[]) => TableKey;
+  urlToPathExpr: (col: string) => string;
+  colRef: (tableKey: TableKey, colName: string) => SQL;
+  tableRef: (tableKey: TableKey) => SQL;
+  fromSql: (tableKey: TableKey, options?: {
+    queryCanonical?: boolean;
+  }) => SQL;
+  dateColRef: (tableKey: TableKey) => SQL;
+  siteIdColRef?: (tableKey: TableKey) => SQL;
+  searchTypeColRef?: (tableKey: TableKey) => SQL;
+  dimExprSql: (dim: Dimension, tableKey: TableKey) => SQL;
+  metricSql: (metric: Metric, tableKey: TableKey) => SQL;
+  havingPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
+  prefilterPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
+  dimensionPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
+  topLevelPredicate: (filters: InternalFilter[], tableKey: TableKey) => SQL | undefined;
+}
+declare function createSqlFragments<TableKey extends string>(config: SqlFragmentsConfig<TableKey>): SqlFragments<TableKey>;
+export { SqlFragments, SqlFragmentsConfig, createSqlFragments };

package/dist/resolver/fragments.mjs

@@ -0,0 +1,206 @@
+import { escapeLike } from "../sql-fragments.mjs";
+import { LOGICAL_DATASETS, UnresolvableDatasetError, inferLogicalDataset, isDatasetResolvable } from "./datasets.mjs";
+import { sql } from "drizzle-orm";
+const METRIC_NAMES = [
+	"clicks",
+	"impressions",
+	"ctr",
+	"position"
+];
+const QUERY_DIM_ALIAS = "query_dim";
+function quoteIdent(id) {
+	return `"${id.replace(/"/g, "\"\"")}"`;
+}
+function qualifiedRaw(alias, column) {
+	return sql.raw(`${quoteIdent(alias)}.${quoteIdent(column)}`);
+}
+function defaultSqliteUrlToPathExpr(col) {
+	return `CASE WHEN ${col} LIKE 'http%' THEN CASE WHEN INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/') > 0 THEN SUBSTR(${col}, INSTR(${col}, '://') + 2 + INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/')) ELSE '/' END ELSE ${col} END`;
+}
+function buildDimensionColumnMap(datasetToTableKey) {
+	const entries = Object.entries(datasetToTableKey).map(([dataset, tableKey]) => {
+		const dims = LOGICAL_DATASETS[dataset].dimensions;
+		return [tableKey, Object.fromEntries(Object.entries(dims).map(([dim, binding]) => [dim, binding?.column ?? dim]))];
+	});
+	return Object.fromEntries(entries);
+}
+function createSqlFragments(config) {
+	const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, includeSearchType, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride, queryCanonicalSource = "queryDim", queryDimTableRef, queryDimSiteScoped = false } = config;
+	const DIM_COLUMN_MAP = buildDimensionColumnMap(datasetToTableKey);
+	function isMetricDimension(dim) {
+		return METRIC_NAMES.includes(dim);
+	}
+	function dimColumn(dim, table) {
+		return DIM_COLUMN_MAP[table]?.[dim] ?? dim;
+	}
+	function tableKeyForDataset(dataset) {
+		return datasetToTableKey[dataset];
+	}
+	function inferTable(dimensions, filterDims = []) {
+		if (!isDatasetResolvable(dimensions, filterDims)) throw new UnresolvableDatasetError(dimensions, filterDims);
+		return tableKeyForDataset(inferLogicalDataset(dimensions, filterDims));
+	}
+	const urlToPathExpr = urlToPathExprOverride ?? defaultSqliteUrlToPathExpr;
+	function colRef(tableKey, colName) {
+		const c = schema[tableKey][colName];
+		if (!c) throw new Error(`${tableLabel}: unknown column '${colName}' on ${tableKey}`);
+		return sql`${c}`;
+	}
+	function tableRef(tableKey) {
+		if (tableRefOverride) return tableRefOverride(tableKey);
+		return sql`${schema[tableKey]}`;
+	}
+	function fromSql(tableKey, options = {}) {
+		const base = tableRef(tableKey);
+		if (!options.queryCanonical || queryCanonicalSource !== "queryDim") return base;
+		return sql`${base} LEFT JOIN ${queryDimTableRef?.() ?? sql.raw(`${quoteIdent(QUERY_DIM_ALIAS)}`)} ON ${queryDimSiteScoped ? sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")} AND ${colRef(tableKey, "site_id")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "site_id")}` : sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")}`}`;
+	}
+	function dateColRef(tableKey) {
+		return colRef(tableKey, "date");
+	}
+	function siteIdColRef(tableKey) {
+		return colRef(tableKey, "site_id");
+	}
+	function searchTypeColRef(tableKey) {
+		return colRef(tableKey, "search_type");
+	}
+	function dimExprSql(dim, tableKey) {
+		const colName = dimColumn(dim, tableKey);
+		if (dim === "page") return sql.raw(urlToPathExpr(colName));
+		if (dim === "queryCanonical") {
+			if (queryCanonicalSource === "queryDim") return sql`COALESCE(${qualifiedRaw(QUERY_DIM_ALIAS, "query_canonical")}, ${colRef(tableKey, "query")})`;
+			return qualifiedRaw(String(tableKey), "query_canonical");
+		}
+		return colRef(tableKey, colName);
+	}
+	function metricSql(metric, tableKey) {
+		const t = schema[tableKey];
+		switch (metric) {
+			case "clicks": return sql`CAST(SUM(${t.clicks}) AS ${sql.raw(metricCast)})`;
+			case "impressions": return sql`CAST(SUM(${t.impressions}) AS ${sql.raw(metricCast)})`;
+			case "ctr": return sql`CAST(SUM(${t.clicks}) AS ${sql.raw(metricCast)}) / NULLIF(SUM(${t.impressions}), 0)`;
+			case "position": return sql`SUM(${t.sum_position}) / NULLIF(SUM(${t.impressions}), 0) + 1`;
+		}
+	}
+	function havingPredicates(filters, tableKey) {
+		const preds = [];
+		for (const f of filters) {
+			const metric = f.dimension;
+			if (!isMetricDimension(metric)) continue;
+			const expr = metricSql(metric, tableKey);
+			const v = Number(f.expression);
+			switch (f.operator) {
+				case "metricGte":
+					preds.push(sql`${expr} >= ${v}`);
+					break;
+				case "metricGt":
+					preds.push(sql`${expr} > ${v}`);
+					break;
+				case "metricLte":
+					preds.push(sql`${expr} <= ${v}`);
+					break;
+				case "metricLt":
+					preds.push(sql`${expr} < ${v}`);
+					break;
+				case "metricBetween": {
+					const v2 = Number(f.expression2);
+					preds.push(sql`${expr} >= ${v} AND ${expr} <= ${v2}`);
+					break;
+				}
+			}
+		}
+		return preds;
+	}
+	function prefilterPredicates(filters, tableKey) {
+		const t = schema[tableKey];
+		const preds = [];
+		for (const f of filters) {
+			const metric = f.dimension;
+			let col;
+			if (metric === "clicks") col = t.clicks;
+			else if (metric === "impressions") col = t.impressions;
+			if (!col) continue;
+			const v = Number(f.expression);
+			switch (f.operator) {
+				case "metricGte":
+					preds.push(sql`${col} >= ${v}`);
+					break;
+				case "metricGt":
+					preds.push(sql`${col} > ${v}`);
+					break;
+				case "metricLte":
+					preds.push(sql`${col} <= ${v}`);
+					break;
+				case "metricLt":
+					preds.push(sql`${col} < ${v}`);
+					break;
+				case "metricBetween": {
+					const v2 = Number(f.expression2);
+					preds.push(sql`${col} >= ${v} AND ${col} <= ${v2}`);
+					break;
+				}
+			}
+		}
+		return preds;
+	}
+	function dimensionPredicates(filters, tableKey) {
+		const preds = [];
+		for (const f of filters) {
+			if (isMetricDimension(f.dimension)) continue;
+			if (f.dimension === "date") continue;
+			if (f.operator === "topLevel") continue;
+			const dim = f.dimension;
+			const cRef = dim === "queryCanonical" ? void 0 : colRef(tableKey, dimColumn(dim, tableKey));
+			const matchExpr = dim === "page" || dim === "queryCanonical" ? dimExprSql(dim, tableKey) : cRef;
+			const patternExpr = dim === "queryCanonical" ? matchExpr : cRef;
+			switch (f.operator) {
+				case "equals":
+					preds.push(sql`${matchExpr} = ${f.expression}`);
+					break;
+				case "notEquals":
+					preds.push(sql`${matchExpr} != ${f.expression}`);
+					break;
+				case "contains":
+					preds.push(sql`${patternExpr} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					break;
+				case "notContains":
+					preds.push(sql`${patternExpr} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					break;
+				case "includingRegex":
+					preds.push(regexPredicate(patternExpr, f.expression, false));
+					break;
+				case "excludingRegex":
+					preds.push(regexPredicate(patternExpr, f.expression, true));
+					break;
+			}
+		}
+		return preds;
+	}
+	function topLevelPredicate(filters, tableKey) {
+		if (!filters.some((f) => f.operator === "topLevel")) return void 0;
+		const pathExpr = dimExprSql("page", tableKey);
+		return sql`LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
+	}
+	return {
+		METRIC_NAMES,
+		DIM_COLUMN_MAP,
+		isMetricDimension,
+		tableKeyForDataset,
+		dimColumn,
+		inferTable,
+		urlToPathExpr,
+		colRef,
+		tableRef,
+		fromSql,
+		dateColRef,
+		siteIdColRef: includeSiteId ? siteIdColRef : void 0,
+		searchTypeColRef: includeSearchType ? searchTypeColRef : void 0,
+		dimExprSql,
+		metricSql,
+		havingPredicates,
+		prefilterPredicates,
+		dimensionPredicates,
+		topLevelPredicate
+	};
+}
+export { createSqlFragments };