@gscdump/engine 0.6.1 → 0.6.3

package/dist/ingest.d.mts

@@ -1,4 +1,4 @@
-import { Row, TableName } from "gscdump/contracts";
+import { N as TableName, w as Row } from "./_chunks/storage.mjs";
 /**
  * Canonical GSC API dimension order per table. Consumers hitting the raw
  * `searchanalytics.query` endpoint must request dimensions in this order so

package/dist/planner.d.mts

@@ -1,16 +1,3 @@
-import { LogicalQueryPlan } from "gscdump/query/plan";
-import { TableName } from "gscdump/contracts";
-import { BuilderState } from "gscdump/query";
-declare function enumeratePartitions(startDate: string, endDate: string): string[];
-interface ResolvedQuery {
-  sql: string;
-  params: unknown[];
-  partitions: string[];
-  table: TableName;
-  filesPlaceholder: string;
-}
-declare const FILES_PLACEHOLDER = "{{FILES}}";
-declare function compileLogicalQueryPlan(plan: LogicalQueryPlan, table?: TableName): ResolvedQuery;
-declare function resolveToSQL(state: BuilderState, table?: TableName): ResolvedQuery;
-declare function substituteNamedFiles(sql: string, sets: Record<string, string[]>): string;
-export { FILES_PLACEHOLDER, type ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };
+import { X as enumeratePartitions } from "./_chunks/storage.mjs";
+import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, r as compileLogicalQueryPlan, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
+export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/dist/planner.mjs

@@ -1,321 +1,2 @@
-import { MS_PER_DAY, toIsoDate } from "gscdump";
-import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
-import { buildLogicalPlan } from "gscdump/query/plan";
-function metricCols() {
-	return {
-		clicks: integer("clicks").notNull(),
-		impressions: integer("impressions").notNull(),
-		sum_position: doublePrecision("sum_position").notNull()
-	};
-}
-const dateCol = () => date("date").notNull();
-const drizzleSchema = {
-	pages: pgTable("pages", {
-		url: varchar("url").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	keywords: pgTable("keywords", {
-		query: varchar("query").notNull(),
-		query_canonical: varchar("query_canonical"),
-		date: dateCol(),
-		...metricCols()
-	}),
-	countries: pgTable("countries", {
-		country: varchar("country").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	devices: pgTable("devices", {
-		device: varchar("device").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	page_keywords: pgTable("page_keywords", {
-		url: varchar("url").notNull(),
-		query: varchar("query").notNull(),
-		query_canonical: varchar("query_canonical"),
-		date: dateCol(),
-		...metricCols()
-	}),
-	search_appearance: pgTable("search_appearance", {
-		searchAppearance: varchar("searchAppearance").notNull(),
-		date: dateCol(),
-		...metricCols()
-	})
-};
-const TABLE_METADATA = {
-	pages: {
-		sortKey: ["date", "url"],
-		version: 1
-	},
-	keywords: {
-		sortKey: ["date", "query"],
-		version: 2
-	},
-	countries: {
-		sortKey: ["date", "country"],
-		version: 1
-	},
-	devices: {
-		sortKey: ["date", "device"],
-		version: 1
-	},
-	page_keywords: {
-		sortKey: [
-			"date",
-			"url",
-			"query"
-		],
-		version: 2
-	},
-	search_appearance: {
-		sortKey: ["date", "searchAppearance"],
-		version: 1
-	}
-};
-function pgSqlTypeToColumnType(sqlType) {
-	const t = sqlType.toLowerCase();
-	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
-	if (t === "date" || t.startsWith("timestamp")) return "DATE";
-	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
-	if (t === "bigint" || t === "int8") return "BIGINT";
-	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
-	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
-}
-function tableSchemaFrom(tableName) {
-	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
-		name: col.name,
-		type: pgSqlTypeToColumnType(col.getSQLType()),
-		nullable: !col.notNull
-	}));
-	const meta = TABLE_METADATA[tableName];
-	return {
-		name: tableName,
-		columns,
-		sortKey: meta.sortKey,
-		version: meta.version
-	};
-}
-Object.fromEntries([
-	"pages",
-	"keywords",
-	"countries",
-	"devices",
-	"page_keywords",
-	"search_appearance"
-].map((t) => [t, tableSchemaFrom(t)]));
-function dimensionToColumn(dim, _table) {
-	if (dim === "page") return "url";
-	if (dim === "queryCanonical") return "query_canonical";
-	return dim;
-}
-function dayPartition(date) {
-	return `daily/${date}`;
-}
-function monthPartition(month) {
-	return `monthly/${month}`;
-}
-function weekPartition(mondayIsoDate) {
-	return `weekly/${mondayIsoDate}`;
-}
-function quarterPartition(quarter) {
-	return `quarterly/${quarter}`;
-}
-function mondayOfWeek(isoDate) {
-	const ms = Date.parse(`${isoDate}T00:00:00Z`);
-	const dow = new Date(ms).getUTCDay();
-	const offset = dow === 0 ? -6 : 1 - dow;
-	return toIsoDate(new Date(ms + offset * MS_PER_DAY));
-}
-function quarterOfMonth(month) {
-	const [y, m] = month.split("-").map(Number);
-	return `${y}-Q${Math.floor((m - 1) / 3) + 1}`;
-}
-const DEFAULT_THRESHOLDS = {
-	raw: 7,
-	d7: 30,
-	d30: 90
-};
-DEFAULT_THRESHOLDS.raw, DEFAULT_THRESHOLDS.d7, DEFAULT_THRESHOLDS.d30;
-function enumeratePartitions(startDate, endDate) {
-	const out = [];
-	const [sy, sm, sd] = startDate.split("-").map(Number);
-	const [ey, em, ed] = endDate.split("-").map(Number);
-	const start = Date.UTC(sy, sm - 1, sd);
-	const end = Date.UTC(ey, em - 1, ed);
-	if (end < start) return out;
-	const seenWeeks = /* @__PURE__ */ new Set();
-	const seenMonths = /* @__PURE__ */ new Set();
-	const seenQuarters = /* @__PURE__ */ new Set();
-	for (let t = start; t <= end; t += 864e5) {
-		const d = new Date(t);
-		const y = d.getUTCFullYear();
-		const m = String(d.getUTCMonth() + 1).padStart(2, "0");
-		const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
-		const isoMonth = `${y}-${m}`;
-		out.push(dayPartition(isoDay));
-		const monday = mondayOfWeek(isoDay);
-		if (!seenWeeks.has(monday)) {
-			seenWeeks.add(monday);
-			out.push(weekPartition(monday));
-		}
-		if (!seenMonths.has(isoMonth)) {
-			seenMonths.add(isoMonth);
-			out.push(monthPartition(isoMonth));
-		}
-		const quarter = quarterOfMonth(isoMonth);
-		if (!seenQuarters.has(quarter)) {
-			seenQuarters.add(quarter);
-			out.push(quarterPartition(quarter));
-		}
-	}
-	return out;
-}
-function escapeLike(value) {
-	return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
-}
-const METRIC_EXPR = {
-	clicks: "CAST(SUM(clicks) AS DOUBLE)",
-	impressions: "CAST(SUM(impressions) AS DOUBLE)",
-	ctr: "CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0)",
-	position: "SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1"
-};
-function topLevelPagePredicateSql(pathExpr) {
-	return `LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
-}
-const FILES_PLACEHOLDER = "{{FILES}}";
-function buildDimensionWhere(filters, table) {
-	const clauses = [];
-	const params = [];
-	for (const filter of filters) {
-		const column = dimensionToColumn(filter.dimension, table);
-		switch (filter.operator) {
-			case "equals":
-				clauses.push(`${column} = ?`);
-				params.push(filter.expression);
-				break;
-			case "notEquals":
-				clauses.push(`${column} != ?`);
-				params.push(filter.expression);
-				break;
-			case "contains":
-				clauses.push(`${column} LIKE ? ESCAPE '\\'`);
-				params.push(`%${escapeLike(filter.expression)}%`);
-				break;
-			case "notContains":
-				clauses.push(`${column} NOT LIKE ? ESCAPE '\\'`);
-				params.push(`%${escapeLike(filter.expression)}%`);
-				break;
-			case "includingRegex":
-				clauses.push(`regexp_matches(${column}, ?)`);
-				params.push(filter.expression);
-				break;
-			case "excludingRegex":
-				clauses.push(`NOT regexp_matches(${column}, ?)`);
-				params.push(filter.expression);
-				break;
-		}
-	}
-	return {
-		clause: clauses.join(" AND "),
-		params
-	};
-}
-function buildTopLevelWhere(plan, table) {
-	if (!plan.specialFilters.topLevel) return "";
-	return topLevelPagePredicateSql(dimensionToColumn("page", table));
-}
-function buildHaving(filters) {
-	if (filters.length === 0) return {
-		clause: "",
-		params: []
-	};
-	const clauses = [];
-	const params = [];
-	for (const filter of filters) {
-		const expr = METRIC_EXPR[filter.metric];
-		switch (filter.operator) {
-			case "metricGte":
-				clauses.push(`${expr} >= ?`);
-				params.push(filter.expression);
-				break;
-			case "metricGt":
-				clauses.push(`${expr} > ?`);
-				params.push(filter.expression);
-				break;
-			case "metricLte":
-				clauses.push(`${expr} <= ?`);
-				params.push(filter.expression);
-				break;
-			case "metricLt":
-				clauses.push(`${expr} < ?`);
-				params.push(filter.expression);
-				break;
-			case "metricBetween":
-				clauses.push(`${expr} >= ? AND ${expr} <= ?`);
-				params.push(filter.expression, filter.expression2 ?? filter.expression);
-				break;
-		}
-	}
-	return {
-		clause: clauses.length > 0 ? `HAVING ${clauses.join(" AND ")}` : "",
-		params
-	};
-}
-function compileLogicalQueryPlan(plan, table = plan.dataset) {
-	const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
-	const metricSelects = plan.metrics.map((metric) => `${METRIC_EXPR[metric]} AS ${metric}`);
-	const dimSelects = plan.groupByDimensions.map((dimension) => {
-		const column = dimensionToColumn(dimension, table);
-		return column !== dimension ? `${column} AS ${dimension}` : dimension;
-	});
-	const whereClauses = ["date >= ?", "date <= ?"];
-	const whereParams = [plan.dateRange.startDate, plan.dateRange.endDate];
-	const dimWhere = buildDimensionWhere(plan.dimensionFilters, table);
-	if (dimWhere.clause) {
-		whereClauses.push(dimWhere.clause);
-		whereParams.push(...dimWhere.params);
-	}
-	const topLevelClause = buildTopLevelWhere(plan, table);
-	if (topLevelClause) whereClauses.push(topLevelClause);
-	const having = buildHaving(plan.metricFilters);
-	const groupByCols = [...plan.groupByDimensions.map((dimension) => dimensionToColumn(dimension, table)), ...plan.hasDate ? ["date"] : []];
-	const groupBy = groupByCols.length > 0 ? `GROUP BY ${groupByCols.join(", ")}` : "";
-	const orderBy = plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC";
-	const limit = `LIMIT ${plan.rowLimit ?? 1e3}`;
-	const offset = plan.startRow ? `OFFSET ${plan.startRow}` : "";
-	return {
-		sql: [
-			`SELECT ${[
-				...dimSelects,
-				...plan.hasDate ? ["date"] : [],
-				...metricSelects
-			].join(", ")}`,
-			`FROM read_parquet(${FILES_PLACEHOLDER}, union_by_name = true)`,
-			`WHERE ${whereClauses.join(" AND ")}`,
-			groupBy,
-			having.clause,
-			orderBy,
-			limit,
-			offset
-		].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(),
-		params: [...whereParams, ...having.params],
-		partitions,
-		table,
-		filesPlaceholder: FILES_PLACEHOLDER
-	};
-}
-function resolveToSQL(state, table) {
-	const plan = buildLogicalPlan(state, { regex: true });
-	return compileLogicalQueryPlan(plan, table ?? plan.dataset);
-}
-function fileList(keys) {
-	return keys.length === 0 ? "[]" : `[${keys.map((key) => `'${key.replace(/'/g, "''")}'`).join(", ")}]`;
-}
-function substituteNamedFiles(sql, sets) {
-	let out = sql;
-	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
-	return out;
-}
+import { i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/compiler.mjs";
 export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/dist/resolver/index.d.mts

@@ -1,7 +1,8 @@
-import { SQL } from "drizzle-orm";
+import { a as ResolvedSQLOptimized, i as ResolvedSQL, n as ExtraQuery, o as ResolverAdapter, r as ResolvedComparisonSQL, s as ResolverOptions, t as ComparisonFilter } from "../_chunks/types.mjs";
 import { LogicalDataset, LogicalDataset as LogicalDataset$1, PlannerCapabilities } from "gscdump/query/plan";
-import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
+import { SQL } from "drizzle-orm";
 import { TableName } from "gscdump/contracts";
+import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
 type DimensionSurface = 'api' | 'stored' | 'derived';
 interface DimensionBinding {
   column: string;
@@ -52,55 +53,6 @@ interface SqlFragments<TableKey extends string> {
   topLevelPredicate: (filters: InternalFilter[], tableKey: TableKey) => SQL | undefined;
 }
 declare function createSqlFragments<TableKey extends string>(config: SqlFragmentsConfig<TableKey>): SqlFragments<TableKey>;
-interface ResolverAdapter<TableKey extends string = string> {
-  readonly METRIC_NAMES: readonly Metric[];
-  readonly capabilities: PlannerCapabilities;
-  readonly schema: Record<TableKey, unknown>;
-  tableKeyForDataset: (dataset: LogicalDataset$1) => TableKey;
-  inferTable: (dimensions: Dimension[], filterDims?: Dimension[]) => TableKey;
-  dimColumn: (dim: Dimension, tableKey: TableKey) => string;
-  isMetricDimension: (dim: string) => dim is Metric;
-  tableRef: (tableKey: TableKey) => SQL;
-  dateColRef: (tableKey: TableKey) => SQL;
-  urlToPathExpr: (col: string) => string;
-  siteIdColRef?: (tableKey: TableKey) => SQL;
-  dimExprSql: (dim: Dimension, tableKey: TableKey) => SQL;
-  metricSql: (metric: Metric, tableKey: TableKey) => SQL;
-  dimensionPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
-  havingPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
-  topLevelPredicate: (filters: InternalFilter[], tableKey: TableKey) => SQL | undefined;
-  compile: (query: SQL) => {
-    sql: string;
-    params: unknown[];
-  };
-}
-type ComparisonFilter = 'new' | 'lost' | 'improving' | 'declining';
-interface ResolverOptions<TableKey extends string = string> {
-  adapter: ResolverAdapter<TableKey>;
-  /** Optional site scope. Required for multi-tenant D1; omitted for parquet. */
-  siteId?: string | number;
-}
-interface ResolvedSQL {
-  sql: string;
-  params: unknown[];
-  countSql: string;
-  countParams: unknown[];
-}
-interface ResolvedSQLOptimized {
-  sql: string;
-  params: unknown[];
-}
-interface ResolvedComparisonSQL {
-  sql: string;
-  params: unknown[];
-  countSql: string;
-  countParams: unknown[];
-}
-interface ExtraQuery {
-  key: string;
-  sql: string;
-  params: unknown[];
-}
 interface CreateResolverAdapterConfig<TableKey extends string> extends SqlFragmentsConfig<TableKey> {
   compile: (query: SQL) => {
     sql: string;