@gscdump/engine 0.11.5 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,667 +0,0 @@
1
- import { u as drizzleSchema } from "./schema.mjs";
2
- import { escapeLike } from "../sql-fragments.mjs";
3
- import { buildLogicalComparisonPlan, buildLogicalPlan } from "gscdump/query/plan";
4
- import { PgDialect } from "drizzle-orm/pg-core";
5
- import { sql } from "drizzle-orm";
6
- const COMPARISON_FILTER_SQL = {
7
- new: sql`AND (p.impressions IS NULL OR p.impressions = 0)`,
8
- lost: sql`AND p.impressions > 0 AND c.impressions = 0`,
9
- improving: sql`AND c.clicks > COALESCE(p.clicks, 0)`,
10
- declining: sql`AND c.clicks < p.clicks AND p.clicks > 0`
11
- };
12
- function collapseWs(s) {
13
- return s.replace(/\s+/g, " ").trim();
14
- }
15
- function joinAnd(parts) {
16
- return sql.join(parts, sql` AND `);
17
- }
18
- function joinComma(parts) {
19
- return sql.join(parts, sql`, `);
20
- }
21
- function orderByClause(state, prefix = "") {
22
- if (state.orderBy) {
23
- const safeCol = state.orderBy.column.replace(/\W/g, "");
24
- const safeDir = state.orderBy.dir.toUpperCase() === "ASC" ? "ASC" : "DESC";
25
- return sql.raw(`ORDER BY ${prefix}${safeCol} ${safeDir}`);
26
- }
27
- return sql.raw(`ORDER BY ${prefix}clicks DESC`);
28
- }
29
- function limitOffsetClause(state) {
30
- const rowLimit = Math.max(0, Math.floor(Number(state.rowLimit ?? 100)));
31
- const offset = state.startRow ? Math.max(0, Math.floor(Number(state.startRow))) : 0;
32
- return sql.raw(offset > 0 ? `LIMIT ${rowLimit} OFFSET ${offset}` : `LIMIT ${rowLimit}`);
33
- }
34
- function aliasRaw(name) {
35
- const safe = name.replace(/\W/g, "");
36
- return sql.raw(`"${safe}"`);
37
- }
38
- function toInternalDimensionFilters(filters) {
39
- return filters.map((filter) => ({
40
- dimension: filter.dimension,
41
- operator: filter.operator,
42
- expression: filter.expression,
43
- expression2: filter.expression2
44
- }));
45
- }
46
- function toInternalMetricFilters(filters) {
47
- return filters.map((filter) => ({
48
- dimension: filter.metric,
49
- operator: filter.operator,
50
- expression: String(filter.expression),
51
- expression2: filter.expression2 == null ? void 0 : String(filter.expression2)
52
- }));
53
- }
54
- function topLevelFilters(plan) {
55
- if (!plan.specialFilters.topLevel) return [];
56
- return [{
57
- dimension: "page",
58
- operator: "topLevel",
59
- expression: ""
60
- }];
61
- }
62
- function logicalFilterToInternal(filter) {
63
- return {
64
- dimension: filter.dimension,
65
- operator: filter.operator,
66
- expression: filter.expression,
67
- expression2: filter.expression2
68
- };
69
- }
70
- function compileFilterTree(node, adapter, tableKey) {
71
- if (!node) return void 0;
72
- if (node.kind === "leaf") return adapter.dimensionPredicates([logicalFilterToInternal(node.filter)], tableKey)[0];
73
- const childSqls = node.children.map((child) => compileFilterTree(child, adapter, tableKey)).filter((s) => s !== void 0);
74
- if (childSqls.length === 0) return void 0;
75
- if (childSqls.length === 1) return childSqls[0];
76
- const sep = node.groupType === "or" ? sql` OR ` : sql` AND `;
77
- return sql`(${sql.join(childSqls, sep)})`;
78
- }
79
- function buildScope(state, options) {
80
- const { adapter, siteId } = options;
81
- const plan = buildLogicalPlan(state, adapter.capabilities);
82
- const tableKey = adapter.tableKeyForDataset(plan.dataset);
83
- const dimFilters = toInternalDimensionFilters(plan.dimensionFilters);
84
- const metricFilters = toInternalMetricFilters(plan.metricFilters);
85
- const groupByDims = plan.groupByDimensions;
86
- const hasDate = plan.hasDate;
87
- const metrics = plan.metrics;
88
- const wherePredicates = [];
89
- if (adapter.siteIdColRef && siteId != null) wherePredicates.push(sql`${adapter.siteIdColRef(tableKey)} = ${siteId}`);
90
- wherePredicates.push(sql`${adapter.dateColRef(tableKey)} >= ${plan.dateRange.startDate}`);
91
- wherePredicates.push(sql`${adapter.dateColRef(tableKey)} <= ${plan.dateRange.endDate}`);
92
- const dimSql = plan.dimensionFilterTree ? compileFilterTree(plan.dimensionFilterTree, adapter, tableKey) : void 0;
93
- if (dimSql) wherePredicates.push(dimSql);
94
- else if (!plan.dimensionFilterTree) wherePredicates.push(...adapter.dimensionPredicates(dimFilters, tableKey));
95
- const tl = adapter.topLevelPredicate(topLevelFilters(plan), tableKey);
96
- if (tl) wherePredicates.push(tl);
97
- return {
98
- plan,
99
- tableKey,
100
- groupByDims,
101
- hasDate,
102
- metrics,
103
- wherePredicates,
104
- having: adapter.havingPredicates(metricFilters, tableKey),
105
- dimFilters,
106
- startDate: plan.dateRange.startDate,
107
- endDate: plan.dateRange.endDate
108
- };
109
- }
110
- function buildComparisonPlan(current, previous, capabilities) {
111
- return buildLogicalComparisonPlan(current, previous, capabilities);
112
- }
113
- function compileCollapsed(adapter, q) {
114
- const c = adapter.compile(q);
115
- return {
116
- sql: collapseWs(c.sql),
117
- params: c.params
118
- };
119
- }
120
- function resolveToSQLOptimized(state, options) {
121
- const { adapter } = options;
122
- const { tableKey, groupByDims, hasDate, metrics, wherePredicates, having } = buildScope(state, options);
123
- const table = adapter.tableRef(tableKey);
124
- const schema = adapter.schema;
125
- const cteSelect = [];
126
- for (const d of groupByDims) {
127
- const expr = adapter.dimExprSql(d, tableKey);
128
- const colName = adapter.dimColumn(d, tableKey);
129
- if (d === "page" || colName !== d) cteSelect.push(sql`${expr} as ${aliasRaw(d)}`);
130
- else cteSelect.push(expr);
131
- }
132
- if (hasDate) cteSelect.push(adapter.dateColRef(tableKey));
133
- const t = schema[tableKey];
134
- cteSelect.push(sql`CAST(SUM(${t.clicks}) AS DOUBLE) as clicks`);
135
- cteSelect.push(sql`CAST(SUM(${t.impressions}) AS DOUBLE) as impressions`);
136
- cteSelect.push(sql`CAST(SUM(${t.sum_position}) AS DOUBLE) as sum_position`);
137
- const groupByExprs = groupByDims.map((d) => adapter.dimExprSql(d, tableKey));
138
- if (hasDate) groupByExprs.push(adapter.dateColRef(tableKey));
139
- const outerSelect = [];
140
- for (const d of groupByDims) outerSelect.push(aliasRaw(d));
141
- if (hasDate) outerSelect.push(sql.raw("date"));
142
- const outerTotals = [];
143
- for (const m of metrics) switch (m) {
144
- case "clicks":
145
- outerSelect.push(sql.raw("clicks"));
146
- outerTotals.push(sql.raw("CAST(SUM(clicks) OVER() AS DOUBLE) as totalClicks"));
147
- break;
148
- case "impressions":
149
- outerSelect.push(sql.raw("impressions"));
150
- outerTotals.push(sql.raw("CAST(SUM(impressions) OVER() AS DOUBLE) as totalImpressions"));
151
- break;
152
- case "ctr":
153
- outerSelect.push(sql.raw("CAST(clicks AS REAL) / NULLIF(impressions, 0) as ctr"));
154
- outerTotals.push(sql.raw("CAST(SUM(clicks) OVER() AS REAL) / NULLIF(SUM(impressions) OVER(), 0) as totalCtr"));
155
- break;
156
- case "position":
157
- outerSelect.push(sql.raw("sum_position / NULLIF(impressions, 0) + 1 as position"));
158
- outerTotals.push(sql.raw("SUM(sum_position) OVER() / NULLIF(SUM(impressions) OVER(), 0) + 1 as totalPosition"));
159
- break;
160
- }
161
- outerSelect.push(sql.raw("COUNT(*) OVER() as totalCount"));
162
- for (const totalExpr of outerTotals) outerSelect.push(totalExpr);
163
- let cte = wherePredicates.length > 0 ? sql`SELECT ${joinComma(cteSelect)} FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT ${joinComma(cteSelect)} FROM ${table}`;
164
- if (groupByExprs.length > 0) cte = sql`${cte} GROUP BY ${joinComma(groupByExprs)}`;
165
- if (having.length > 0) cte = sql`${cte} HAVING ${joinAnd(having)}`;
166
- return compileCollapsed(adapter, sql`WITH aggregated AS (${cte}) SELECT ${joinComma(outerSelect)} FROM aggregated ${orderByClause(state)} ${limitOffsetClause(state)}`);
167
- }
168
- function resolveToSQL(state, options) {
169
- const { adapter } = options;
170
- const { tableKey, groupByDims, hasDate, metrics, wherePredicates, having } = buildScope(state, options);
171
- const table = adapter.tableRef(tableKey);
172
- const selectExprs = [];
173
- for (const d of groupByDims) {
174
- const expr = adapter.dimExprSql(d, tableKey);
175
- const colName = adapter.dimColumn(d, tableKey);
176
- if (d === "page" || colName !== d) selectExprs.push(sql`${expr} as ${aliasRaw(d)}`);
177
- else selectExprs.push(expr);
178
- }
179
- if (hasDate) selectExprs.push(adapter.dateColRef(tableKey));
180
- for (const m of metrics) selectExprs.push(sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`);
181
- const groupByExprs = groupByDims.map((d) => adapter.dimExprSql(d, tableKey));
182
- if (hasDate) groupByExprs.push(adapter.dateColRef(tableKey));
183
- let body = wherePredicates.length > 0 ? sql`SELECT ${joinComma(selectExprs)} FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT ${joinComma(selectExprs)} FROM ${table}`;
184
- if (groupByExprs.length > 0) body = sql`${body} GROUP BY ${joinComma(groupByExprs)}`;
185
- if (having.length > 0) body = sql`${body} HAVING ${joinAnd(having)}`;
186
- const mainQuery = sql`${body} ${orderByClause(state)} ${limitOffsetClause(state)}`;
187
- let countQuery;
188
- if (groupByExprs.length > 0) {
189
- let inner = wherePredicates.length > 0 ? sql`SELECT ${joinComma(groupByExprs)} FROM ${table} WHERE ${joinAnd(wherePredicates)} GROUP BY ${joinComma(groupByExprs)}` : sql`SELECT ${joinComma(groupByExprs)} FROM ${table} GROUP BY ${joinComma(groupByExprs)}`;
190
- if (having.length > 0) inner = sql`${inner} HAVING ${joinAnd(having)}`;
191
- countQuery = sql`SELECT COUNT(*) as total FROM (${inner})`;
192
- } else countQuery = wherePredicates.length > 0 ? sql`SELECT COUNT(*) as total FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT COUNT(*) as total FROM ${table}`;
193
- const main = compileCollapsed(adapter, mainQuery);
194
- const count = compileCollapsed(adapter, countQuery);
195
- return {
196
- sql: main.sql,
197
- params: main.params,
198
- countSql: count.sql,
199
- countParams: count.params
200
- };
201
- }
202
- function buildTotalsSql(state, options) {
203
- const { adapter } = options;
204
- const { tableKey, metrics, wherePredicates } = buildScope(state, options);
205
- const table = adapter.tableRef(tableKey);
206
- const selectExprs = metrics.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`);
207
- return compileCollapsed(adapter, wherePredicates.length > 0 ? sql`SELECT ${joinComma(selectExprs)} FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT ${joinComma(selectExprs)} FROM ${table}`);
208
- }
209
- function resolveComparisonSQL(current, previous, options, comparisonFilter) {
210
- const { adapter, siteId } = options;
211
- const comparisonPlan = buildComparisonPlan(current, previous, adapter.capabilities);
212
- const currentScope = buildScope(current, options);
213
- const previousScope = buildScope(previous, options);
214
- const { tableKey, groupByDims, metrics, wherePredicates: currentWhere, having } = currentScope;
215
- const table = adapter.tableRef(tableKey);
216
- const dimSelectExprs = [];
217
- for (const d of groupByDims) {
218
- const expr = adapter.dimExprSql(d, tableKey);
219
- const colName = adapter.dimColumn(d, tableKey);
220
- if (d === "page" || colName !== d) dimSelectExprs.push(sql`${expr} as ${aliasRaw(d)}`);
221
- else dimSelectExprs.push(expr);
222
- }
223
- const currentSelect = [...dimSelectExprs, ...metrics.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`)];
224
- const prevSelect = [...dimSelectExprs, ...adapter.METRIC_NAMES.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`)];
225
- const groupByExprs = groupByDims.map((d) => adapter.dimExprSql(d, tableKey));
226
- const prevWhere = [];
227
- if (adapter.siteIdColRef && siteId != null) prevWhere.push(sql`${adapter.siteIdColRef(tableKey)} = ${siteId}`);
228
- if (previousScope.startDate) prevWhere.push(sql`${adapter.dateColRef(tableKey)} >= ${previousScope.startDate}`);
229
- if (previousScope.endDate) prevWhere.push(sql`${adapter.dateColRef(tableKey)} <= ${previousScope.endDate}`);
230
- const prevDimSql = comparisonPlan.current.dimensionFilterTree ? compileFilterTree(comparisonPlan.current.dimensionFilterTree, adapter, tableKey) : void 0;
231
- if (prevDimSql) prevWhere.push(prevDimSql);
232
- else if (!comparisonPlan.current.dimensionFilterTree) prevWhere.push(...adapter.dimensionPredicates(toInternalDimensionFilters(comparisonPlan.current.dimensionFilters), tableKey));
233
- let currentCte = currentWhere.length > 0 ? sql`SELECT ${joinComma(currentSelect)} FROM ${table} WHERE ${joinAnd(currentWhere)}` : sql`SELECT ${joinComma(currentSelect)} FROM ${table}`;
234
- if (groupByExprs.length > 0) currentCte = sql`${currentCte} GROUP BY ${joinComma(groupByExprs)}`;
235
- if (having.length > 0) currentCte = sql`${currentCte} HAVING ${joinAnd(having)}`;
236
- let previousCte = prevWhere.length > 0 ? sql`SELECT ${joinComma(prevSelect)} FROM ${table} WHERE ${joinAnd(prevWhere)}` : sql`SELECT ${joinComma(prevSelect)} FROM ${table}`;
237
- if (groupByExprs.length > 0) previousCte = sql`${previousCte} GROUP BY ${joinComma(groupByExprs)}`;
238
- const joinOn = groupByDims.length > 0 ? sql.raw(groupByDims.map((d) => `c.${d.replace(/\W/g, "")} = p.${d.replace(/\W/g, "")}`).join(" AND ")) : sql.raw("1=1");
239
- const filterClause = comparisonFilter ? COMPARISON_FILTER_SQL[comparisonFilter] : sql.raw("");
240
- const orderSql = orderByClause(current, "c.");
241
- const limitSql = limitOffsetClause(current);
242
- const outerCurrentCols = [];
243
- for (const d of groupByDims) {
244
- const colName = d.replace(/\W/g, "");
245
- outerCurrentCols.push(sql.raw(`c.${colName} as "${colName}"`));
246
- }
247
- outerCurrentCols.push(sql.raw("CAST(c.clicks AS DOUBLE) as \"clicks\""));
248
- outerCurrentCols.push(sql.raw("CAST(c.impressions AS DOUBLE) as \"impressions\""));
249
- outerCurrentCols.push(sql.raw("c.ctr as \"ctr\""));
250
- outerCurrentCols.push(sql.raw("c.position as \"position\""));
251
- const mainQuery = sql`WITH current AS (${currentCte}), previous AS (${previousCte}) SELECT ${joinComma(outerCurrentCols)}, COALESCE(CAST(p.clicks AS DOUBLE), 0) as "prevClicks", COALESCE(CAST(p.impressions AS DOUBLE), 0) as "prevImpressions", COALESCE(p.ctr, 0) as "prevCtr", COALESCE(p.position, 0) as "prevPosition" FROM current c LEFT JOIN previous p ON ${joinOn} WHERE 1=1 ${filterClause} ${orderSql} ${limitSql}`;
252
- const firstGroupBy = groupByDims[0] ? groupByDims[0].replace(/\W/g, "") : "clicks";
253
- const countInnerSelect = sql.raw(`c.${firstGroupBy}`);
254
- const countQuery = sql`WITH current AS (${currentCte}), previous AS (${previousCte}) SELECT COUNT(*) as total FROM (SELECT ${countInnerSelect} FROM current c LEFT JOIN previous p ON ${joinOn} WHERE 1=1 ${filterClause})`;
255
- const main = compileCollapsed(adapter, mainQuery);
256
- const count = compileCollapsed(adapter, countQuery);
257
- return {
258
- sql: main.sql,
259
- params: main.params,
260
- countSql: count.sql,
261
- countParams: count.params
262
- };
263
- }
264
- function buildExtrasQueries(state, options) {
265
- const { adapter, siteId } = options;
266
- const plan = buildLogicalPlan(state, adapter.capabilities);
267
- const dims = plan.groupByDimensions;
268
- const extras = [];
269
- if (!dims.includes("queryCanonical")) return extras;
270
- const keywordsKey = adapter.tableKeyForDataset("keywords");
271
- const t = adapter.schema[keywordsKey];
272
- const table = adapter.tableRef(keywordsKey);
273
- const whereParts = [];
274
- if (adapter.siteIdColRef && siteId != null) whereParts.push(sql`${adapter.siteIdColRef(keywordsKey)} = ${siteId}`);
275
- whereParts.push(sql`${adapter.dateColRef(keywordsKey)} >= ${plan.dateRange.startDate}`);
276
- whereParts.push(sql`${adapter.dateColRef(keywordsKey)} <= ${plan.dateRange.endDate}`);
277
- const whereExpr = whereParts.length > 0 ? sql`WHERE ${joinAnd(whereParts)}` : sql``;
278
- const outerQueryCol = sql.raw("query");
279
- const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${t.query_canonical} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${t.query_canonical} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${t.query_canonical}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${t.query_canonical}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
280
- extras.push({
281
- key: "canonicalExtras",
282
- sql: compiled.sql,
283
- params: compiled.params
284
- });
285
- return extras;
286
- }
287
- function mergeExtras(rows, extrasResults) {
288
- if (extrasResults.length === 0) return rows;
289
- const lookups = [];
290
- for (const { key, results } of extrasResults) {
291
- if (key === "canonicalExtras") {
292
- const variantCountMap = /* @__PURE__ */ new Map();
293
- const variantsMap = /* @__PURE__ */ new Map();
294
- const canonicalNameMap = /* @__PURE__ */ new Map();
295
- for (const r of results) {
296
- const jk = String(r.joinKey);
297
- variantCountMap.set(jk, r.variantCount);
298
- canonicalNameMap.set(jk, r.canonicalName);
299
- const raw = r.variants;
300
- variantsMap.set(jk, typeof raw === "string" ? raw.split("||").filter(Boolean).map((v) => {
301
- const parts = v.split(":::");
302
- return {
303
- query: parts[0],
304
- clicks: Number(parts[1] || 0),
305
- impressions: Number(parts[2] || 0),
306
- position: Number(parts[3] || 0)
307
- };
308
- }) : []);
309
- }
310
- lookups.push({
311
- key: "variantCount",
312
- map: variantCountMap
313
- });
314
- lookups.push({
315
- key: "variants",
316
- map: variantsMap
317
- });
318
- lookups.push({
319
- key: "canonicalName",
320
- map: canonicalNameMap
321
- });
322
- continue;
323
- }
324
- const filtered = results.filter((r) => r.rn === void 0 || r.rn === 1);
325
- const map = /* @__PURE__ */ new Map();
326
- for (const r of filtered) {
327
- let val = r[key];
328
- if (key === "variants" && typeof val === "string") val = val.split("||").filter(Boolean).map((v) => {
329
- const parts = v.split(":::");
330
- return {
331
- query: parts[0],
332
- clicks: Number(parts[1] || 0),
333
- impressions: Number(parts[2] || 0),
334
- position: Number(parts[3] || 0)
335
- };
336
- });
337
- map.set(String(r.joinKey), val);
338
- }
339
- lookups.push({
340
- key,
341
- map
342
- });
343
- }
344
- return rows.map((row) => {
345
- const enriched = { ...row };
346
- for (const { key, map } of lookups) {
347
- let joinValue;
348
- if (key === "variantCount" || key === "variants" || key === "canonicalName") joinValue = String(row.queryCanonical ?? row.query_canonical ?? "");
349
- enriched[key] = (joinValue && map.get(joinValue)) ?? (key === "variants" ? [] : null);
350
- if (key === "canonicalName" && enriched[key]) enriched.queryCanonical = enriched[key];
351
- }
352
- return enriched;
353
- });
354
- }
355
- const DIMENSION_SURFACES = {
356
- page: ["api", "stored"],
357
- query: ["api", "stored"],
358
- queryCanonical: ["stored", "derived"],
359
- country: ["api", "stored"],
360
- device: ["api", "stored"],
361
- searchAppearance: ["api", "stored"],
362
- date: ["api", "stored"]
363
- };
364
- const LOGICAL_DATASETS = {
365
- pages: { dimensions: {
366
- page: {
367
- column: "url",
368
- surfaces: ["api", "stored"]
369
- },
370
- date: {
371
- column: "date",
372
- surfaces: ["api", "stored"]
373
- }
374
- } },
375
- keywords: { dimensions: {
376
- query: {
377
- column: "query",
378
- surfaces: ["api", "stored"]
379
- },
380
- queryCanonical: {
381
- column: "query_canonical",
382
- surfaces: ["stored", "derived"]
383
- },
384
- date: {
385
- column: "date",
386
- surfaces: ["api", "stored"]
387
- }
388
- } },
389
- page_keywords: { dimensions: {
390
- page: {
391
- column: "url",
392
- surfaces: ["api", "stored"]
393
- },
394
- query: {
395
- column: "query",
396
- surfaces: ["api", "stored"]
397
- },
398
- queryCanonical: {
399
- column: "query_canonical",
400
- surfaces: ["stored", "derived"]
401
- },
402
- date: {
403
- column: "date",
404
- surfaces: ["api", "stored"]
405
- }
406
- } },
407
- countries: { dimensions: {
408
- country: {
409
- column: "country",
410
- surfaces: ["api", "stored"]
411
- },
412
- date: {
413
- column: "date",
414
- surfaces: ["api", "stored"]
415
- }
416
- } },
417
- devices: { dimensions: {
418
- device: {
419
- column: "device",
420
- surfaces: ["api", "stored"]
421
- },
422
- date: {
423
- column: "date",
424
- surfaces: ["api", "stored"]
425
- }
426
- } },
427
- search_appearance: { dimensions: {
428
- searchAppearance: {
429
- column: "searchAppearance",
430
- surfaces: ["api", "stored"]
431
- },
432
- date: {
433
- column: "date",
434
- surfaces: ["api", "stored"]
435
- }
436
- } }
437
- };
438
- function inferLogicalDataset(dimensions, filterDims = []) {
439
- const allDims = new Set([...dimensions, ...filterDims]);
440
- const has = (d) => allDims.has(d);
441
- if (has("searchAppearance")) return "search_appearance";
442
- if (has("page") && (has("query") || has("queryCanonical"))) return "page_keywords";
443
- if (has("query") || has("queryCanonical")) return "keywords";
444
- if (has("page")) return "pages";
445
- if (has("country")) return "countries";
446
- if (has("device")) return "devices";
447
- return "keywords";
448
- }
449
- function dimensionColumn(dim, dataset) {
450
- return LOGICAL_DATASETS[dataset].dimensions[dim]?.column ?? dim;
451
- }
452
- function supportsDimensionOnSurface(dim, surface) {
453
- return DIMENSION_SURFACES[dim].includes(surface);
454
- }
455
- function assertDimensionsSupported(dimensions, surface, context) {
456
- const unsupported = dimensions.filter((dim) => !supportsDimensionOnSurface(dim, surface));
457
- if (unsupported.length === 0) return;
458
- throw new Error(`${context}: unsupported dimensions for ${surface}: ${unsupported.join(", ")}`);
459
- }
460
- const METRIC_NAMES = [
461
- "clicks",
462
- "impressions",
463
- "ctr",
464
- "position"
465
- ];
466
- function defaultSqliteUrlToPathExpr(col) {
467
- return `CASE WHEN ${col} LIKE 'http%' THEN CASE WHEN INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/') > 0 THEN SUBSTR(${col}, INSTR(${col}, '://') + 2 + INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/')) ELSE '/' END ELSE ${col} END`;
468
- }
469
- function buildDimensionColumnMap(datasetToTableKey) {
470
- const entries = Object.entries(datasetToTableKey).map(([dataset, tableKey]) => {
471
- const dims = LOGICAL_DATASETS[dataset].dimensions;
472
- return [tableKey, Object.fromEntries(Object.entries(dims).map(([dim, binding]) => [dim, binding?.column ?? dim]))];
473
- });
474
- return Object.fromEntries(entries);
475
- }
476
- function createSqlFragments(config) {
477
- const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride } = config;
478
- const DIM_COLUMN_MAP = buildDimensionColumnMap(datasetToTableKey);
479
- function isMetricDimension(dim) {
480
- return METRIC_NAMES.includes(dim);
481
- }
482
- function dimColumn(dim, table) {
483
- return DIM_COLUMN_MAP[table]?.[dim] ?? dim;
484
- }
485
- function tableKeyForDataset(dataset) {
486
- return datasetToTableKey[dataset];
487
- }
488
- function inferTable(dimensions, filterDims = []) {
489
- return tableKeyForDataset(inferLogicalDataset(dimensions, filterDims));
490
- }
491
- const urlToPathExpr = urlToPathExprOverride ?? defaultSqliteUrlToPathExpr;
492
- function colRef(tableKey, colName) {
493
- const c = schema[tableKey][colName];
494
- if (!c) throw new Error(`${tableLabel}: unknown column '${colName}' on ${tableKey}`);
495
- return sql`${c}`;
496
- }
497
- function tableRef(tableKey) {
498
- if (tableRefOverride) return tableRefOverride(tableKey);
499
- return sql`${schema[tableKey]}`;
500
- }
501
- function dateColRef(tableKey) {
502
- return colRef(tableKey, "date");
503
- }
504
- function siteIdColRef(tableKey) {
505
- return colRef(tableKey, "site_id");
506
- }
507
- function dimExprSql(dim, tableKey) {
508
- const colName = dimColumn(dim, tableKey);
509
- if (dim === "page") return sql.raw(urlToPathExpr(colName));
510
- return colRef(tableKey, colName);
511
- }
512
- function metricSql(metric, tableKey) {
513
- const t = schema[tableKey];
514
- switch (metric) {
515
- case "clicks": return sql`CAST(SUM(${t.clicks}) AS ${sql.raw(metricCast)})`;
516
- case "impressions": return sql`CAST(SUM(${t.impressions}) AS ${sql.raw(metricCast)})`;
517
- case "ctr": return sql`CAST(SUM(${t.clicks}) AS ${sql.raw(metricCast)}) / NULLIF(SUM(${t.impressions}), 0)`;
518
- case "position": return sql`SUM(${t.sum_position}) / NULLIF(SUM(${t.impressions}), 0) + 1`;
519
- }
520
- }
521
- function havingPredicates(filters, tableKey) {
522
- const preds = [];
523
- for (const f of filters) {
524
- const metric = f.dimension;
525
- if (!isMetricDimension(metric)) continue;
526
- const expr = metricSql(metric, tableKey);
527
- const v = Number(f.expression);
528
- switch (f.operator) {
529
- case "metricGte":
530
- preds.push(sql`${expr} >= ${v}`);
531
- break;
532
- case "metricGt":
533
- preds.push(sql`${expr} > ${v}`);
534
- break;
535
- case "metricLte":
536
- preds.push(sql`${expr} <= ${v}`);
537
- break;
538
- case "metricLt":
539
- preds.push(sql`${expr} < ${v}`);
540
- break;
541
- case "metricBetween": {
542
- const v2 = Number(f.expression2);
543
- preds.push(sql`${expr} >= ${v} AND ${expr} <= ${v2}`);
544
- break;
545
- }
546
- }
547
- }
548
- return preds;
549
- }
550
- function dimensionPredicates(filters, tableKey) {
551
- const preds = [];
552
- for (const f of filters) {
553
- if (isMetricDimension(f.dimension)) continue;
554
- if (f.dimension === "date") continue;
555
- if (f.operator === "topLevel") continue;
556
- const cRef = colRef(tableKey, dimColumn(f.dimension, tableKey));
557
- const matchExpr = f.dimension === "page" ? dimExprSql(f.dimension, tableKey) : cRef;
558
- switch (f.operator) {
559
- case "equals":
560
- preds.push(sql`${matchExpr} = ${f.expression}`);
561
- break;
562
- case "notEquals":
563
- preds.push(sql`${matchExpr} != ${f.expression}`);
564
- break;
565
- case "contains":
566
- preds.push(sql`${cRef} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
567
- break;
568
- case "notContains":
569
- preds.push(sql`${cRef} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
570
- break;
571
- case "includingRegex":
572
- preds.push(regexPredicate(cRef, f.expression, false));
573
- break;
574
- case "excludingRegex":
575
- preds.push(regexPredicate(cRef, f.expression, true));
576
- break;
577
- }
578
- }
579
- return preds;
580
- }
581
- function topLevelPredicate(filters, tableKey) {
582
- if (!filters.some((f) => f.operator === "topLevel")) return void 0;
583
- const pathExpr = dimExprSql("page", tableKey);
584
- return sql`LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
585
- }
586
- return {
587
- METRIC_NAMES,
588
- DIM_COLUMN_MAP,
589
- isMetricDimension,
590
- tableKeyForDataset,
591
- dimColumn,
592
- inferTable,
593
- urlToPathExpr,
594
- colRef,
595
- tableRef,
596
- dateColRef,
597
- siteIdColRef: includeSiteId ? siteIdColRef : void 0,
598
- dimExprSql,
599
- metricSql,
600
- havingPredicates,
601
- dimensionPredicates,
602
- topLevelPredicate
603
- };
604
- }
605
- function createResolverAdapter(config) {
606
- const runtime = createSqlFragments(config);
607
- return {
608
- METRIC_NAMES: runtime.METRIC_NAMES,
609
- capabilities: config.capabilities,
610
- schema: config.schema,
611
- tableKeyForDataset: runtime.tableKeyForDataset,
612
- inferTable: runtime.inferTable,
613
- dimColumn: runtime.dimColumn,
614
- isMetricDimension: runtime.isMetricDimension,
615
- tableRef: runtime.tableRef,
616
- dateColRef: runtime.dateColRef,
617
- urlToPathExpr: runtime.urlToPathExpr,
618
- siteIdColRef: runtime.siteIdColRef,
619
- dimExprSql: runtime.dimExprSql,
620
- metricSql: runtime.metricSql,
621
- dimensionPredicates: runtime.dimensionPredicates,
622
- havingPredicates: runtime.havingPredicates,
623
- topLevelPredicate: runtime.topLevelPredicate,
624
- compile: config.compile
625
- };
626
- }
627
- const pgDialect = new PgDialect();
628
- function compilePg(query) {
629
- const compiled = pgDialect.sqlToQuery(query);
630
- return {
631
- sql: compiled.sql,
632
- params: compiled.params
633
- };
634
- }
635
- const PG_BASE_CONFIG = {
636
- schema: drizzleSchema,
637
- datasetToTableKey: {
638
- pages: "pages",
639
- keywords: "keywords",
640
- page_keywords: "page_keywords",
641
- countries: "countries",
642
- devices: "devices",
643
- search_appearance: "search_appearance"
644
- },
645
- metricCast: "DOUBLE",
646
- regexPredicate: (expr, pattern, negate) => negate ? sql`NOT regexp_matches(${expr}, ${pattern})` : sql`regexp_matches(${expr}, ${pattern})`,
647
- urlToPathExpr: (col) => `CASE WHEN ${col} LIKE 'http%' THEN COALESCE(NULLIF(regexp_replace(${col}, '^https?://[^/]+', ''), ''), '/') ELSE ${col} END`,
648
- includeSiteId: false,
649
- compile: compilePg,
650
- capabilities: {
651
- regex: true,
652
- comparisonJoin: true,
653
- windowTotals: true
654
- }
655
- };
656
- const pgResolverAdapter = createResolverAdapter({
657
- ...PG_BASE_CONFIG,
658
- tableLabel: "pg-resolver-adapter"
659
- });
660
- function createParquetResolverAdapter() {
661
- return createResolverAdapter({
662
- ...PG_BASE_CONFIG,
663
- tableLabel: "parquet-resolver-adapter",
664
- tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`)
665
- });
666
- }
667
- export { DIMENSION_SURFACES as a, dimensionColumn as c, buildExtrasQueries as d, buildTotalsSql as f, resolveToSQLOptimized as g, resolveToSQL as h, createSqlFragments as i, inferLogicalDataset as l, resolveComparisonSQL as m, pgResolverAdapter as n, LOGICAL_DATASETS as o, mergeExtras as p, createResolverAdapter as r, assertDimensionsSupported as s, createParquetResolverAdapter as t, supportsDimensionOnSurface as u };