@gscdump/engine 0.20.3 → 0.21.1

package/dist/_chunks/engine.mjs

@@ -264,7 +264,7 @@ async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
 	}
 	return { deleted: retired.length + sweptOrphans + hourlyDeleted };
 }
-const URL_PURGE_TABLES = ["pages", "page_keywords"];
+const URL_PURGE_TABLES = ["pages", "page_queries"];
 const MAX_DAY_BYTES = 100 * 1024 * 1024;
 const URL_COLUMNS = /* @__PURE__ */ new Set();
 for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);

package/dist/_chunks/iceberg-schema.mjs

@@ -0,0 +1,67 @@
+import { t as SCHEMAS } from "./schema.mjs";
+const ICEBERG_TABLES = [
+	"pages",
+	"queries",
+	"countries",
+	"page_queries",
+	"dates"
+];
+const ICEBERG_PARTITION_COLUMNS = [{
+	name: "site_id",
+	type: "STRING",
+	required: true,
+	fieldId: 1
+}, {
+	name: "search_type",
+	type: "STRING",
+	required: true,
+	fieldId: 2
+}];
+const ICEBERG_FIELD_ID_BASE = 3;
+const ICEBERG_PARTITION_SPEC = [
+	{
+		sourceColumn: "site_id",
+		transform: "identity",
+		name: "site_id"
+	},
+	{
+		sourceColumn: "search_type",
+		transform: "identity",
+		name: "search_type"
+	},
+	{
+		sourceColumn: "date",
+		transform: "month",
+		name: "date_month"
+	}
+];
+function mapColumnType(t) {
+	switch (t) {
+		case "VARCHAR": return "STRING";
+		case "INTEGER": return "INT";
+		case "BIGINT": return "LONG";
+		case "DOUBLE": return "DOUBLE";
+		case "DATE": return "DATE";
+	}
+}
+function icebergTableSpec(table) {
+	const base = SCHEMAS[table];
+	const dataColumns = base.columns.map((col, i) => ({
+		name: col.name,
+		type: mapColumnType(col.type),
+		required: !col.nullable,
+		fieldId: 3 + i
+	}));
+	return {
+		table,
+		columns: [...ICEBERG_PARTITION_COLUMNS, ...dataColumns],
+		partitionSpec: ICEBERG_PARTITION_SPEC,
+		identityColumns: [
+			"site_id",
+			"search_type",
+			...base.sortKey
+		]
+	};
+}
+const ICEBERG_SCHEMAS = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t)]));
+export { ICEBERG_TABLES as a, ICEBERG_SCHEMAS as i, ICEBERG_PARTITION_COLUMNS as n, icebergTableSpec as o, ICEBERG_PARTITION_SPEC as r, ICEBERG_FIELD_ID_BASE as t };

package/dist/_chunks/registry.d.mts

@@ -120,7 +120,7 @@ interface ReduceContext<TRow extends Row$1 = Row$1> {
 }
 /**
  * Unified analyzer contract. `TRow` lets authors narrow from the default
- * `Row = Record<string, unknown>` to a typed row shape (e.g. `KeywordRow`)
+ * `Row = Record<string, unknown>` to a typed row shape (e.g. `QueriesRow`)
  * when their reducer assumes specific columns exist — catches drift between
  * `build` (SELECT list) and `reduce` (column access) at compile time.
  */

package/dist/_chunks/resolver.mjs

@@ -27,7 +27,7 @@ const LOGICAL_DATASETS = {
 			surfaces: ["api", "stored"]
 		}
 	} },
-	keywords: { dimensions: {
+	queries: { dimensions: {
 		query: {
 			column: "query",
 			surfaces: ["api", "stored"]
@@ -41,7 +41,7 @@ const LOGICAL_DATASETS = {
 			surfaces: ["api", "stored"]
 		}
 	} },
-	page_keywords: { dimensions: {
+	page_queries: { dimensions: {
 		page: {
 			column: "url",
 			surfaces: ["api", "stored"]
@@ -69,16 +69,10 @@ const LOGICAL_DATASETS = {
 			surfaces: ["api", "stored"]
 		}
 	} },
-	devices: { dimensions: {
-		device: {
-			column: "device",
-			surfaces: ["api", "stored"]
-		},
-		date: {
-			column: "date",
-			surfaces: ["api", "stored"]
-		}
-	} },
+	dates: { dimensions: { date: {
+		column: "date",
+		surfaces: ["api", "stored"]
+	} } },
 	search_appearance: { dimensions: {
 		searchAppearance: {
 			column: "searchAppearance",
@@ -584,13 +578,13 @@ function buildExtrasQueries(state, options) {
 	const dims = plan.groupByDimensions;
 	const extras = [];
 	if (!dims.includes("queryCanonical")) return extras;
-	const keywordsKey = adapter.tableKeyForDataset("keywords");
-	const t = adapter.schema[keywordsKey];
-	const table = adapter.tableRef(keywordsKey);
+	const queriesKey = adapter.tableKeyForDataset("queries");
+	const t = adapter.schema[queriesKey];
+	const table = adapter.tableRef(queriesKey);
 	const whereParts = [];
-	if (adapter.siteIdColRef && siteId != null) whereParts.push(sql`${adapter.siteIdColRef(keywordsKey)} = ${siteId}`);
-	whereParts.push(sql`${adapter.dateColRef(keywordsKey)} >= ${plan.dateRange.startDate}`);
-	whereParts.push(sql`${adapter.dateColRef(keywordsKey)} <= ${plan.dateRange.endDate}`);
+	if (adapter.siteIdColRef && siteId != null) whereParts.push(sql`${adapter.siteIdColRef(queriesKey)} = ${siteId}`);
+	whereParts.push(sql`${adapter.dateColRef(queriesKey)} >= ${plan.dateRange.startDate}`);
+	whereParts.push(sql`${adapter.dateColRef(queriesKey)} <= ${plan.dateRange.endDate}`);
 	const whereExpr = whereParts.length > 0 ? sql`WHERE ${joinAnd(whereParts)}` : sql``;
 	const outerQueryCol = sql.raw("query");
 	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${t.query_canonical} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${t.query_canonical} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${t.query_canonical}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${t.query_canonical}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
@@ -735,10 +729,10 @@ const PG_BASE_CONFIG = {
 	schema: drizzleSchema,
 	datasetToTableKey: {
 		pages: "pages",
-		keywords: "keywords",
-		page_keywords: "page_keywords",
+		queries: "queries",
+		page_queries: "page_queries",
 		countries: "countries",
-		devices: "devices",
+		dates: "dates",
 		search_appearance: "search_appearance",
 		hourly_pages: "hourly_pages"
 	},