@gscdump/cli 0.37.3 → 0.37.4

package/dist/_chunks/query.mjs

@@ -0,0 +1,537 @@
+import { loadConfig } from "./config.mjs";
+import { ALL_SEARCH_TYPES, exportToCSV, logger, parseSearchType } from "./utils.mjs";
+import { allTables, createCommandContext, inferTable } from "./context.mjs";
+import { gscErrorHandler } from "./error-handler.mjs";
+import process from "node:process";
+import { defineCommand } from "citty";
+import fs from "node:fs/promises";
+import { cancel, isCancel, multiselect, text } from "@clack/prompts";
+import { and, between, contains, country, date, device, eq, gsc, hour, notRegex, page, query, regex, searchAppearance } from "gscdump/query";
+import { daysAgoUtc } from "gscdump/dates";
+import { collectSpans } from "@gscdump/engine";
+const DIMENSIONS = [
+	"page",
+	"query",
+	"date",
+	"hour",
+	"country",
+	"device",
+	"searchAppearance"
+];
+const DIM_COLUMNS = {
+	page,
+	query,
+	date,
+	hour,
+	country,
+	device,
+	searchAppearance
+};
+const FILTER_DIMS = [
+	"query",
+	"page",
+	"country",
+	"device",
+	"searchAppearance"
+];
+const FILTER_COL = {
+	query,
+	page,
+	country,
+	device,
+	searchAppearance
+};
+const DATA_STATES = [
+	"all",
+	"final",
+	"hourly_all"
+];
+const AGGREGATION_TYPES = [
+	"auto",
+	"byPage",
+	"byProperty"
+];
+function buildFilterFromArg(col, raw) {
+	if (raw.startsWith("!~")) return makeLeaf(col, "notContains", raw.slice(2));
+	if (raw.startsWith("!re:")) return notRegex(col, raw.slice(4));
+	if (raw.startsWith("!")) return makeLeaf(col, "notEquals", raw.slice(1));
+	if (raw.startsWith("~")) return contains(col, raw.slice(1));
+	if (raw.startsWith("re:")) return regex(col, raw.slice(3));
+	if (raw.startsWith("contains:")) return contains(col, raw.slice(9));
+	if (raw.startsWith("eq:")) return eq(col, raw.slice(3));
+	return eq(col, raw);
+}
+function makeLeaf(col, operator, value) {
+	return {
+		_constraints: {},
+		_filters: [{
+			dimension: col.dimension,
+			operator,
+			expression: value
+		}]
+	};
+}
+async function runLiveQuery(client, siteUrl, opts) {
+	const allRows = [];
+	const totalLimit = Math.max(0, opts.rowLimit);
+	const pageSize = Math.min(totalLimit, 25e3);
+	let startRow = 0;
+	const baseBody = {
+		startDate: opts.startDate,
+		endDate: opts.endDate,
+		dimensions: opts.dimensions
+	};
+	if (opts.searchType) baseBody.type = opts.searchType;
+	if (opts.dataState) baseBody.dataState = opts.dataState;
+	if (opts.aggregationType) baseBody.aggregationType = opts.aggregationType;
+	if (opts.dimensionFilter) {
+		const groups = filterToGroups(opts.dimensionFilter);
+		if (groups.length > 0) baseBody.dimensionFilterGroups = groups;
+	}
+	while (allRows.length < totalLimit) {
+		const remaining = totalLimit - allRows.length;
+		const rowLimit = Math.min(pageSize, remaining);
+		if (rowLimit <= 0) break;
+		const rows = ((await client._rawQuery(siteUrl, {
+			...baseBody,
+			rowLimit,
+			startRow
+		})).rows || []).map((row) => {
+			const result = {
+				clicks: row.clicks ?? 0,
+				impressions: row.impressions ?? 0,
+				ctr: row.ctr ?? 0,
+				position: row.position ?? 0
+			};
+			opts.dimensions.forEach((dim, i) => {
+				result[dim] = row.keys?.[i];
+			});
+			return result;
+		});
+		if (rows.length === 0) break;
+		allRows.push(...rows);
+		startRow += rows.length;
+	}
+	return { rows: allRows };
+}
+function filterToGroups(f) {
+	if (f._filters.length === 0) return [];
+	return [{ filters: f._filters.map((leaf) => ({
+		dimension: leaf.dimension,
+		operator: leaf.operator,
+		expression: leaf.expression
+	})) }];
+}
+const queryCommand = defineCommand({
+	meta: {
+		name: "query",
+		description: "Run a search analytics query (local Parquet by default, --live hits GSC API)"
+	},
+	args: {
+		"site": {
+			type: "string",
+			alias: "s",
+			description: "Site URL (e.g., sc-domain:example.com)"
+		},
+		"dimensions": {
+			type: "string",
+			alias: "d",
+			description: `Dimensions: ${DIMENSIONS.join(",")}`
+		},
+		"start": {
+			type: "string",
+			description: "Start date (YYYY-MM-DD)"
+		},
+		"end": {
+			type: "string",
+			description: "End date (YYYY-MM-DD)"
+		},
+		"limit": {
+			type: "string",
+			alias: "l",
+			default: "1000",
+			description: "Max rows (default: 1000)"
+		},
+		"output": {
+			type: "string",
+			alias: "o",
+			description: "Output file path (default: stdout)"
+		},
+		"format": {
+			type: "string",
+			alias: "f",
+			default: "json",
+			description: "Output format: json or csv"
+		},
+		"sql": {
+			type: "string",
+			description: "Raw DuckDB SQL using {{FILES}} as the file list placeholder (bypasses builder)"
+		},
+		"table": {
+			type: "string",
+			description: "Analytics table for --sql (default: pages)"
+		},
+		"live": {
+			type: "boolean",
+			default: false,
+			description: "Bypass local store; hit the GSC API directly"
+		},
+		"quiet": {
+			type: "boolean",
+			alias: "q",
+			default: false,
+			description: "Suppress progress output"
+		},
+		"interactive": {
+			type: "boolean",
+			alias: "i",
+			default: false,
+			description: "Interactive mode"
+		},
+		"query": {
+			type: "string",
+			description: "Filter by query (prefix: ~contains, !exclude, re:regex, !re:not-regex)"
+		},
+		"page": {
+			type: "string",
+			description: "Filter by page (same prefix syntax as --query)"
+		},
+		"country": {
+			type: "string",
+			description: "Filter by country (ISO-3 lowercase, e.g. usa). Same prefix syntax as --query"
+		},
+		"device": {
+			type: "string",
+			description: "Filter by device (DESKTOP/MOBILE/TABLET). Same prefix syntax"
+		},
+		"search-appearance": {
+			type: "string",
+			description: "Filter by search appearance feature. Same prefix syntax"
+		},
+		"type": {
+			type: "string",
+			description: `Search type (live mode only). One of: ${ALL_SEARCH_TYPES.join(",")}`
+		},
+		"data-state": {
+			type: "string",
+			description: `Data state (live mode only). One of: ${DATA_STATES.join(",")} (default: final)`
+		},
+		"aggregation-type": {
+			type: "string",
+			description: `Aggregation type (live mode only). One of: ${AGGREGATION_TYPES.join(",")}`
+		},
+		"explain": {
+			type: "boolean",
+			default: false,
+			description: "Print the request body / planned local SQL and exit without executing"
+		},
+		"profile": {
+			type: "boolean",
+			default: false,
+			description: "Print a query timing breakdown (manifest list / file fetch / SQL run) to stderr (local mode only)"
+		}
+	},
+	async run({ args }) {
+		if (args.sql) {
+			await runRawSqlMode({
+				sql: String(args.sql),
+				site: args.site ? String(args.site) : void 0,
+				table: args.table ? String(args.table) : "pages",
+				output: args.output ? String(args.output) : void 0,
+				quiet: Boolean(args.quiet),
+				searchType: parseSearchType(args.type, "--type")
+			});
+			return;
+		}
+		const ctxConfig = await loadConfig();
+		const dimNames = await resolveDimensions(args);
+		const { startDate, endDate } = await resolveRange(args);
+		await promptFilters(args);
+		const limitArg = args.limit != null ? String(args.limit) : null;
+		const rowLimit = limitArg != null && limitArg !== "1000" ? Number.parseInt(limitArg, 10) : ctxConfig.defaultLimit ?? 1e3;
+		const format = String(args.format);
+		const dimensionFilter = buildDimensionFilter(args);
+		const searchType = parseSearchType(args.type ?? ctxConfig.defaultSearchType, "--type");
+		const dataState = args["data-state"] ? String(args["data-state"]) : ctxConfig.defaultDataState;
+		const aggregationType = args["aggregation-type"] ? String(args["aggregation-type"]) : void 0;
+		if (dataState && !DATA_STATES.includes(dataState)) {
+			logger.error(`Invalid --data-state: ${dataState}. Allowed: ${DATA_STATES.join(", ")}`);
+			process.exit(1);
+		}
+		if (aggregationType && !AGGREGATION_TYPES.includes(aggregationType)) {
+			logger.error(`Invalid --aggregation-type: ${aggregationType}. Allowed: ${AGGREGATION_TYPES.join(", ")}`);
+			process.exit(1);
+		}
+		const ctx = await createCommandContext({
+			needsAuth: true,
+			needsStore: !args.live,
+			interactive: Boolean(args.interactive)
+		});
+		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+		if (args.live) {
+			if (args.explain) {
+				const body = {
+					startDate,
+					endDate,
+					dimensions: dimNames,
+					rowLimit
+				};
+				if (searchType) body.type = searchType;
+				if (dataState) body.dataState = dataState;
+				if (aggregationType) body.aggregationType = aggregationType;
+				if (dimensionFilter) body.dimensionFilterGroups = filterToGroups(dimensionFilter);
+				console.log(JSON.stringify({
+					siteUrl,
+					body
+				}, null, 2));
+				return;
+			}
+			if (!args.quiet) logger.info(`Querying ${siteUrl} via live GSC API...`);
+			const result = await runLiveQuery(ctx.client, siteUrl, {
+				startDate,
+				endDate,
+				dimensions: dimNames,
+				rowLimit,
+				searchType,
+				dataState,
+				aggregationType,
+				dimensionFilter
+			}).catch(gscErrorHandler);
+			await writeOutput({
+				output: {
+					siteUrl,
+					dimensions: dimNames,
+					dateRange: {
+						start: startDate,
+						end: endDate
+					},
+					total: result.rows.length,
+					data: result.rows
+				},
+				format,
+				path: args.output ? String(args.output) : void 0,
+				quiet: Boolean(args.quiet)
+			});
+			return;
+		}
+		if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
+		if (!args.quiet) logger.info(`Querying ${siteUrl} from local Parquet store...`);
+		const state = buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter);
+		const store = ctx.store;
+		const table = inferTable(dimNames);
+		if (args.explain) {
+			console.log(JSON.stringify({
+				siteUrl,
+				table,
+				state
+			}, null, 2));
+			return;
+		}
+		await assertRangeCovered(store, siteUrl, table, startDate, endDate, searchType);
+		const probe = Boolean(args.profile) ? collectSpans() : void 0;
+		const result = await store.engine.query({
+			userId: store.userId,
+			siteId: store.siteIdFor(siteUrl),
+			table,
+			...searchType !== void 0 ? { searchType } : {},
+			...probe ? { profiler: probe.profiler } : {}
+		}, state).catch((e) => {
+			logger.error(`Query failed: ${e.message}`);
+			process.exit(1);
+		});
+		if (probe) logProfile(probe.spans);
+		await writeOutput({
+			output: {
+				siteUrl,
+				dimensions: dimNames,
+				dateRange: {
+					start: startDate,
+					end: endDate
+				},
+				total: result.rows.length,
+				data: result.rows
+			},
+			format,
+			path: args.output ? String(args.output) : void 0,
+			quiet: Boolean(args.quiet)
+		});
+	}
+});
+async function resolveDimensions(args) {
+	if (args.dimensions) return String(args.dimensions).split(",").filter((d) => DIMENSIONS.includes(d));
+	if (args.interactive) {
+		const selected = await multiselect({
+			message: "Select dimensions",
+			options: DIMENSIONS.map((d) => ({
+				value: d,
+				label: d
+			})),
+			initialValues: ["page", "query"]
+		});
+		if (isCancel(selected)) {
+			cancel("Cancelled");
+			process.exit(0);
+		}
+		return selected;
+	}
+	return ["page", "query"];
+}
+async function resolveRange(args) {
+	if (args.start && args.end) return {
+		startDate: String(args.start),
+		endDate: String(args.end)
+	};
+	if (args.interactive) {
+		const startInput = await text({
+			message: "Start date (YYYY-MM-DD)",
+			placeholder: daysAgoUtc(28)
+		});
+		if (isCancel(startInput)) {
+			cancel("Cancelled");
+			process.exit(0);
+		}
+		const endInput = await text({
+			message: "End date (YYYY-MM-DD)",
+			placeholder: daysAgoUtc(3)
+		});
+		if (isCancel(endInput)) {
+			cancel("Cancelled");
+			process.exit(0);
+		}
+		return {
+			startDate: String(startInput) || daysAgoUtc(28),
+			endDate: String(endInput) || daysAgoUtc(3)
+		};
+	}
+	return {
+		startDate: daysAgoUtc(31),
+		endDate: daysAgoUtc(3)
+	};
+}
+async function promptFilters(args) {
+	if (!args.interactive) return;
+	for (const dim of FILTER_DIMS) {
+		if (args[dim]) continue;
+		const v = await text({
+			message: `Filter by ${dim} (blank to skip; prefix ~ for contains, ! for not-equals, re: for regex)`,
+			placeholder: ""
+		});
+		if (isCancel(v)) {
+			cancel("Cancelled");
+			process.exit(0);
+		}
+		if (v && String(v).length > 0) args[dim] = String(v);
+	}
+	if (!args.type) {
+		const t = await text({
+			message: `Search type (blank for default web; allowed: ${ALL_SEARCH_TYPES.join(", ")})`,
+			placeholder: ""
+		});
+		if (isCancel(t)) {
+			cancel("Cancelled");
+			process.exit(0);
+		}
+		if (t && String(t).length > 0) args.type = String(t);
+	}
+	if (!args["data-state"]) {
+		const ds = await text({
+			message: `Data state (blank for default 'final'; allowed: ${DATA_STATES.join(", ")})`,
+			placeholder: ""
+		});
+		if (isCancel(ds)) {
+			cancel("Cancelled");
+			process.exit(0);
+		}
+		if (ds && String(ds).length > 0) args["data-state"] = String(ds);
+	}
+}
+function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
+	const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
+	const dateFilter = between(date, startDate, endDate);
+	const filter = dimensionFilter ? and(dateFilter, dimensionFilter) : dateFilter;
+	return gsc.select(...dims).where(filter).limit(rowLimit).getState();
+}
+function buildDimensionFilter(args) {
+	const leaves = [];
+	for (const dim of FILTER_DIMS) {
+		const raw = args[dim];
+		if (raw == null || raw === "") continue;
+		leaves.push(buildFilterFromArg(FILTER_COL[dim], String(raw)));
+	}
+	if (leaves.length === 0) return void 0;
+	if (leaves.length === 1) return leaves[0];
+	return and(...leaves);
+}
+async function assertRangeCovered(store, siteUrl, table, startDate, endDate, searchType) {
+	const wm = (await store.engine.getWatermarks({
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl),
+		table,
+		...searchType !== void 0 ? { searchType } : {}
+	}))[0];
+	if (!wm) {
+		logger.error(`No data synced for ${siteUrl} / ${table}. Run \`gscdump sync\` first, or pass --live.`);
+		process.exit(1);
+	}
+	if (endDate > wm.newestDateSynced) {
+		logger.error(`Requested end=${endDate} is newer than last sync (${wm.newestDateSynced}). Run \`gscdump sync\` first, or pass --live.`);
+		process.exit(1);
+	}
+	if (startDate < wm.oldestDateSynced) {
+		logger.error(`Requested start=${startDate} is older than first sync (${wm.oldestDateSynced}). Run \`gscdump sync --start=${startDate}\` first, or pass --live.`);
+		process.exit(1);
+	}
+}
+async function runRawSqlMode(opts) {
+	if (!isKnownTable(opts.table)) {
+		logger.error(`Unknown table "${opts.table}". Known: ${allTables().join(", ")}`);
+		process.exit(1);
+	}
+	const ctx = await createCommandContext({
+		needsAuth: true,
+		needsStore: true
+	});
+	const siteUrl = await ctx.resolveSite(opts.site);
+	const store = ctx.store;
+	if (!opts.quiet) logger.info(`Running raw SQL over table "${opts.table}" for ${siteUrl}`);
+	const { rows, sql } = await store.runRawSql({
+		sql: opts.sql,
+		siteUrl,
+		table: opts.table,
+		...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+	}).catch((e) => {
+		logger.error(`SQL failed: ${e.message}`);
+		process.exit(1);
+	});
+	const payload = JSON.stringify({
+		sql,
+		total: rows.length,
+		data: rows
+	}, null, 2);
+	if (opts.output && opts.output !== "-") {
+		await fs.writeFile(opts.output, payload);
+		if (!opts.quiet) logger.info(`Written to ${opts.output}`);
+	} else console.log(payload);
+}
+function logProfile(spans) {
+	if (spans.length === 0) {
+		logger.warn("No profiling spans recorded (the executor may not be instrumented).");
+		return;
+	}
+	const fmtMeta = (m) => m ? Object.entries(m).map(([k, v]) => `${k}=${v}`).join(" ") : "";
+	const row = (name, ms, meta) => `  ${name.padEnd(18)} ${`${ms}ms`.padStart(8)}  ${fmtMeta(meta)}`.trimEnd();
+	const lines = spans.map((s) => row(s.name, s.ms, s.meta));
+	const total = spans.filter((s) => s.name === "manifest.list" || s.name === "executor.execute").reduce((n, s) => n + s.ms, 0);
+	logger.info(`Query timing breakdown:\n${lines.join("\n")}\n${row("total", total)}`);
+}
+async function writeOutput(opts) {
+	const content = opts.format === "csv" ? exportToCSV(opts.output) : JSON.stringify(opts.output, null, 2);
+	if (opts.path && opts.path !== "-") {
+		await fs.writeFile(opts.path, content);
+		if (!opts.quiet) logger.info(`Written to ${opts.path}`);
+	} else console.log(content);
+}
+function isKnownTable(name) {
+	return allTables().includes(name);
+}
+export { queryCommand };