@gscdump/cli 0.29.0 → 0.31.0

package/dist/index.mjs

@@ -4,7 +4,7 @@ import { defineCommand, runMain } from "citty";
 import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
 import fs, { readFile, readdir, rm } from "node:fs/promises";
 import path, { join } from "node:path";
-import { AnalyzerCapabilityError, createEngineQuerySource, runAnalyzerFromSource } from "@gscdump/analysis";
+import { AnalyzerCapabilityError, INTENT_CLASSIFIER_VERSION, NORMALIZER_VERSION, classifyQueryIntent, createEngineQuerySource, encodeIntent, normalizeQuery, runAnalyzerFromSource } from "@gscdump/analysis";
 import { createGscApiQuerySource } from "@gscdump/engine-gsc-api";
 import { err, ok, unwrapResult } from "gscdump/result";
 import { decodeSiteId, normalizeSiteUrl } from "gscdump/tenant";
@@ -26,7 +26,7 @@ import { isEngineError } from "@gscdump/engine/errors";
 import { DuckDBInstance } from "@duckdb/node-api";
 import { sqlEscape } from "@gscdump/engine/sql";
 import { dateReplaceClause } from "@gscdump/engine/sql-fragments";
-import { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore } from "@gscdump/engine/entities";
+import { buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore } from "@gscdump/engine/entities";
 import { daysAgo, getDateRange, progressBar } from "gscdump";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -34,7 +34,7 @@ import { z } from "zod";
 import { defaultReportRegistry, dryRunReport, formatReport, runReport } from "@gscdump/analysis/report";
 import { resolveWindow } from "@gscdump/engine/period";
 import { collectSpans, inferLegacyTier } from "@gscdump/engine";
-import { DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/engine/rollups";
+import { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/engine/rollups";
 import { filesystemStats } from "@gscdump/engine/filesystem";
 var __defProp = Object.defineProperty;
 var __exportAll = (all, no_symbols) => {
@@ -136,7 +136,7 @@ function loadEnvFromCwd() {
 	}
 	return applied;
 }
-var version = "0.29.0";
+var version = "0.31.0";
 const ALL_SEARCH_TYPES$1 = Object.values(SearchTypes);
 const VERSION = version;
 function noSubcommandSelected(parent, subNames) {
@@ -6347,6 +6347,35 @@ const gcCommand = defineCommand({
 		logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
 	}
 });
+async function buildSiteQueryDim(store, siteId) {
+	const ctx = {
+		userId: store.userId,
+		siteId
+	};
+	const entries = await store.engine.listLive({
+		userId: ctx.userId,
+		siteId,
+		table: "queries"
+	});
+	if (entries.length === 0) return 0;
+	const { rows } = await store.engine.runSQL({
+		ctx,
+		table: "queries",
+		fileSets: { FILES: {
+			table: "queries",
+			partitions: entries.map((e) => e.partition)
+		} },
+		sql: `SELECT DISTINCT query FROM read_parquet({{FILES}}, union_by_name = true) WHERE query IS NOT NULL`
+	});
+	const records = buildQueryDimRecords(rows.map((r) => String(r.query)), {
+		normalizeQuery,
+		normalizerVersion: NORMALIZER_VERSION,
+		classifyIntentCode: (q) => encodeIntent(classifyQueryIntent(q)),
+		intentVersion: INTENT_CLASSIFIER_VERSION
+	});
+	await createQueryDimStore({ dataSource: store.dataSource }).write(ctx, records, Date.now());
+	return records.length;
+}
 const rollupsCommand = defineCommand({
 	meta: {
 		name: "rollups",
@@ -6359,14 +6388,20 @@ const rollupsCommand = defineCommand({
 		},
 		args: {
 			...OUTPUT_ARGS,
-			site: {
+			"site": {
 				type: "string",
 				alias: "s",
 				description: "Restrict to a single site (default: all sites with local data)"
+			},
+			"with-canonical": {
+				type: "boolean",
+				description: "Also build the opt-in canonical-primary rollups (query_canonical_variants, query_canonical_daily)",
+				default: false
 			}
 		},
 		async run({ args }) {
 			const { json } = applyOutputMode(args);
+			const defs = args["with-canonical"] ? [...DEFAULT_ROLLUPS, ...CANONICAL_ROLLUPS] : DEFAULT_ROLLUPS;
 			const store = (await createCommandContext({ needsStore: true })).store;
 			const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
 			const allSiteIds = /* @__PURE__ */ new Set();
@@ -6389,7 +6424,11 @@ const rollupsCommand = defineCommand({
 			const summary = [];
 			let totalBytes = 0;
 			for (const siteId of allSiteIds) {
-				logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)`);
+				if (args["with-canonical"]) {
+					const dimRows = await buildSiteQueryDim(store, siteId);
+					if (!json) logger.info(`Built query dimension for [${siteId}] (${dimRows} distinct queries, normalizer v${NORMALIZER_VERSION})`);
+				}
+				logger.info(`Rebuilding rollups for [${siteId}] (${defs.length} rollups)`);
 				const results = await rebuildRollups({
 					engine: {
 						runSQL: (opts) => store.engine.runSQL(opts),
@@ -6410,7 +6449,7 @@ const rollupsCommand = defineCommand({
 						userId: store.userId,
 						siteId
 					},
-					defs: DEFAULT_ROLLUPS
+					defs
 				});
 				const site = {
 					siteId,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/cli",
   "type": "module",
-  "version": "0.29.0",
+  "version": "0.31.0",
   "description": "CLI for Google Search Console - dump, query, and run MCP server",
   "author": {
     "name": "Harlan Wilton",
@@ -43,10 +43,10 @@
     "ofetch": "^1.5.1",
     "open": "^11.0.0",
     "zod": "^4.4.3",
-    "@gscdump/analysis": "0.29.0",
-    "@gscdump/engine-gsc-api": "0.29.0",
-    "@gscdump/engine": "0.29.0",
-    "gscdump": "0.29.0"
+    "@gscdump/analysis": "0.31.0",
+    "@gscdump/engine": "0.31.0",
+    "@gscdump/engine-gsc-api": "0.31.0",
+    "gscdump": "0.31.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "1.5.1-r.2",