@gscdump/engine 0.30.0 → 0.31.0

package/dist/_chunks/entities.mjs

@@ -16,6 +16,105 @@ async function readOptional(ds, key, signal) {
 		throw e;
 	});
 }
+const QUERY_DIM_COLUMNS = [
+	{
+		name: "query",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "query_canonical",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "intent_code",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "normalizer_version",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "intent_version",
+		type: "INTEGER",
+		nullable: false
+	}
+];
+function queryDimPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/query_dim` : `u_${ctx.userId}/entities/query_dim`;
+}
+function queryDimParquetKey(ctx) {
+	return `${queryDimPrefix(ctx)}/index.parquet`;
+}
+function queryDimMetaKey(ctx) {
+	return `${queryDimPrefix(ctx)}/index.json`;
+}
+function buildQueryDimRecords(queries, deps) {
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (const raw of queries) {
+		const query = String(raw);
+		if (query.trim() === "" || seen.has(query)) continue;
+		seen.add(query);
+		const canonical = deps.normalizeQuery(query);
+		out.push({
+			query,
+			query_canonical: canonical === "" ? query : canonical,
+			intent_code: deps.classifyIntentCode(query),
+			normalizer_version: deps.normalizerVersion,
+			intent_version: deps.intentVersion
+		});
+	}
+	return out;
+}
+function createQueryDimStore({ dataSource }) {
+	async function exists(key, prefix) {
+		return (await dataSource.list(prefix)).includes(key);
+	}
+	return {
+		parquetKey: queryDimParquetKey,
+		async write(ctx, records, builtAt) {
+			const parquetKey = queryDimParquetKey(ctx);
+			const bytes = encodeRowsToParquetFlex(records, {
+				columns: QUERY_DIM_COLUMNS,
+				sortKey: ["query"]
+			});
+			await dataSource.write(parquetKey, bytes);
+			const meta = {
+				version: 1,
+				builtAt,
+				rowCount: records.length,
+				normalizerVersion: records[0]?.normalizer_version ?? 0,
+				intentVersion: records[0]?.intent_version ?? 0
+			};
+			await dataSource.write(queryDimMetaKey(ctx), new TextEncoder().encode(JSON.stringify(meta)));
+			return {
+				parquetKey,
+				rowCount: records.length
+			};
+		},
+		async loadMeta(ctx) {
+			const key = queryDimMetaKey(ctx);
+			if (!await exists(key, `${queryDimPrefix(ctx)}/`)) return null;
+			const bytes = await dataSource.read(key);
+			return JSON.parse(new TextDecoder().decode(bytes));
+		},
+		async loadRecords(ctx) {
+			const key = queryDimParquetKey(ctx);
+			if (!await exists(key, `${queryDimPrefix(ctx)}/`)) return [];
+			return (await decodeParquetToRows(await dataSource.read(key))).map((r) => ({
+				query: String(r.query),
+				query_canonical: String(r.query_canonical),
+				intent_code: Number(r.intent_code),
+				normalizer_version: Number(r.normalizer_version),
+				intent_version: Number(r.intent_version)
+			}));
+		}
+	};
+}
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 function inspectionIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
@@ -893,4 +992,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/_chunks/libs/icebird.d.mts

@@ -133,7 +133,7 @@ interface Snapshot {
   'sequence-number': number;
   'timestamp-ms': number;
   'manifest-list': string;
-  manifests?: Manifest$1[];
+  manifests?: Manifest[];
   summary: {
     // spec: "value of these fields should be of string type"
     operation: string; // 'spark.app.id'?: string
@@ -192,7 +192,7 @@ interface MetadataLog {
   'timestamp-ms': number;
   'metadata-file': string;
 }
-interface Manifest$1 {
+interface Manifest {
   manifest_path: string;
   manifest_length: bigint;
   partition_spec_id: number;

package/dist/_chunks/resolver.mjs

@@ -297,8 +297,10 @@ function createSqlFragments(config) {
 			if (isMetricDimension(f.dimension)) continue;
 			if (f.dimension === "date") continue;
 			if (f.operator === "topLevel") continue;
-			const cRef = colRef(tableKey, dimColumn(f.dimension, tableKey));
-			const matchExpr = f.dimension === "page" ? dimExprSql(f.dimension, tableKey) : cRef;
+			const dim = f.dimension;
+			const cRef = colRef(tableKey, dimColumn(dim, tableKey));
+			const matchExpr = dim === "page" || dim === "queryCanonical" ? dimExprSql(dim, tableKey) : cRef;
+			const patternExpr = dim === "queryCanonical" ? matchExpr : cRef;
 			switch (f.operator) {
 				case "equals":
 					preds.push(sql`${matchExpr} = ${f.expression}`);
@@ -307,16 +309,16 @@ function createSqlFragments(config) {
 					preds.push(sql`${matchExpr} != ${f.expression}`);
 					break;
 				case "contains":
-					preds.push(sql`${cRef} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					preds.push(sql`${patternExpr} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
 					break;
 				case "notContains":
-					preds.push(sql`${cRef} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					preds.push(sql`${patternExpr} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
 					break;
 				case "includingRegex":
-					preds.push(regexPredicate(cRef, f.expression, false));
+					preds.push(regexPredicate(patternExpr, f.expression, false));
 					break;
 				case "excludingRegex":
-					preds.push(regexPredicate(cRef, f.expression, true));
+					preds.push(regexPredicate(patternExpr, f.expression, true));
 					break;
 			}
 		}

package/dist/entities.d.mts

@@ -1,6 +1,60 @@
 import { DataSource } from "./_chunks/storage.mjs";
 import { ScheduleState } from "./schedule.mjs";
 import { ColumnDef, TenantCtx } from "@gscdump/contracts";
+interface QueryDimRecord {
+  query: string;
+  /** Lexical canonical, never empty: NULL/'' folds to the raw query. */
+  query_canonical: string;
+  /** Packed search-intent code (see `@gscdump/analysis` `encodeIntent`). */
+  intent_code: number;
+  normalizer_version: number;
+  intent_version: number;
+}
+/** JSON sidecar: versions + freshness, readable without decoding the parquet. */
+interface QueryDimMeta {
+  version: 1;
+  builtAt: number;
+  rowCount: number;
+  normalizerVersion: number;
+  intentVersion: number;
+}
+declare function queryDimParquetKey(ctx: TenantCtx): string;
+declare function queryDimMetaKey(ctx: TenantCtx): string;
+/**
+ * Injected derivation. `engine` never imports `@gscdump/analysis`; the host
+ * passes `normalizeQuery` / `classifyIntentCode` (e.g. `encodeIntent ∘
+ * classifyQueryIntent`) plus their version constants.
+ */
+interface QueryDimDeps {
+  normalizeQuery: (query: string) => string;
+  normalizerVersion: number;
+  /** Returns the packed intent code for a raw query. */
+  classifyIntentCode: (query: string) => number;
+  intentVersion: number;
+}
+/**
+ * Pure: distinct raw queries → dimension records. De-dupes, drops empties, and
+ * folds an empty/whitespace canonical back to the raw query so the key is
+ * total (matches the read path's `COALESCE(NULLIF(query_canonical, ''), query)`).
+ */
+declare function buildQueryDimRecords(queries: Iterable<string>, deps: QueryDimDeps): QueryDimRecord[];
+interface QueryDimStore {
+  parquetKey: (ctx: TenantCtx) => string;
+  /** Write the parquet + JSON sidecar. Last-write-wins; no history. */
+  write: (ctx: TenantCtx, records: readonly QueryDimRecord[], builtAt: number) => Promise<{
+    parquetKey: string;
+    rowCount: number;
+  }>;
+  /** Read the sidecar (versions + freshness), or null on first build. */
+  loadMeta: (ctx: TenantCtx) => Promise<QueryDimMeta | null>;
+  /** Decode the dimension rows (test/inspection; reads JOIN the parquet by key). */
+  loadRecords: (ctx: TenantCtx) => Promise<QueryDimRecord[]>;
+}
+declare function createQueryDimStore({
+  dataSource
+}: {
+  dataSource: DataSource;
+}): QueryDimStore;
 /**
  * GSC URL inspection result fields we persist. Mirrors the
  *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
@@ -442,4 +496,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/entities.mjs

@@ -1,2 +1,2 @@
-import { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
-export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+import { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
+export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/rollups.d.mts

@@ -244,6 +244,12 @@ declare function runWindowed(opts: {
     start: string;
     end: string;
   }) => string;
+  /**
+   * Extra named file sets merged into every window's `runSQL` (alongside the
+   * windowed `FILES`). Use to JOIN a non-windowed sidecar (e.g. the query
+   * dimension parquet via `{ QUERY_DIM: { keys: [...] } }`) inside `sqlFor`.
+   */
+  extraFileSets?: Record<string, FileSetRef>;
 }): Promise<Row$1[]>;
 /**
  * Daily totals across the full history. One row per (date, table) with

package/dist/rollups.mjs

@@ -1,7 +1,7 @@
 import "./_chunks/layout.mjs";
 import { engineErrors } from "./errors.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
+import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx, searchType) {
 	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -237,10 +237,13 @@ async function runWindowed(opts) {
 		const result = await opts.engine.runSQL({
 			ctx: opts.ctx,
 			table: opts.table,
-			fileSets: { FILES: {
-				table: opts.table,
-				partitions: w.partitions
-			} },
+			fileSets: {
+				FILES: {
+					table: opts.table,
+					partitions: w.partitions
+				},
+				...opts.extraFileSets
+			},
 			sql: opts.sqlFor(w),
 			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
 		});
@@ -638,23 +641,41 @@ const queryCanonicalDailyRollup = {
 		}
 	],
 	parquetSortKey: ["date", "query_canonical"],
-	async build({ engine, ctx, searchType }) {
+	async build({ engine, ctx, dataSource, searchType }) {
+		const dimStore = createQueryDimStore({ dataSource });
+		const useDim = await dimStore.loadMeta(ctx) !== null;
+		const canonExpr = useDim ? `COALESCE(qd.query_canonical, NULLIF(q.query_canonical, ''), q.query)` : `COALESCE(NULLIF(query_canonical, ''), query)`;
 		return (await runWindowed({
 			engine,
 			ctx,
 			table: "queries",
 			...searchType !== void 0 ? { searchType } : {},
-			sqlFor: (w) => `
-        SELECT
-          COALESCE(NULLIF(query_canonical, ''), query) AS query_canonical,
-          CAST(date AS VARCHAR) AS date,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${w.start}' AND date <= '${w.end}'
-        GROUP BY COALESCE(NULLIF(query_canonical, ''), query), date
-      `
+			...useDim ? { extraFileSets: { QUERY_DIM: {
+				table: "queries",
+				keys: [dimStore.parquetKey(ctx)]
+			} } } : {},
+			sqlFor: useDim ? (w) => `
+          SELECT
+            ${canonExpr} AS query_canonical,
+            CAST(q.date AS VARCHAR) AS date,
+            SUM(q.clicks)::BIGINT AS clicks,
+            SUM(q.impressions)::BIGINT AS impressions,
+            SUM(q.sum_position)::DOUBLE AS sum_position
+          FROM read_parquet({{FILES}}, union_by_name = true) q
+          LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query
+          WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'
+          GROUP BY ${canonExpr}, q.date
+        ` : (w) => `
+          SELECT
+            ${canonExpr} AS query_canonical,
+            CAST(date AS VARCHAR) AS date,
+            SUM(clicks)::BIGINT AS clicks,
+            SUM(impressions)::BIGINT AS impressions,
+            SUM(sum_position)::DOUBLE AS sum_position
+          FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE date >= '${w.start}' AND date <= '${w.end}'
+          GROUP BY ${canonExpr}, date
+        `
 		})).map((r) => ({
 			query_canonical: String(r.query_canonical),
 			date: String(r.date),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.30.0",
+  "version": "0.31.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -191,8 +191,8 @@
     "hyparquet": "^1.26.1",
     "hyparquet-writer": "^0.16.1",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.30.0",
-    "gscdump": "0.30.0"
+    "@gscdump/contracts": "0.31.0",
+    "gscdump": "0.31.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",