@gscdump/cli 0.37.2 → 0.37.4

package/dist/_chunks/entities.mjs

@@ -0,0 +1,414 @@
+import { OUTPUT_ARGS, applyOutputMode, logger, runWithConcurrency } from "./utils.mjs";
+import { createCommandContext } from "./context.mjs";
+import process from "node:process";
+import { defineCommand } from "citty";
+import { readFile } from "node:fs/promises";
+import { Buffer } from "node:buffer";
+import { createIndexingMetadataStore, createInspectionStore, createSitemapStore } from "@gscdump/engine/entities";
+import { progressBar } from "gscdump";
+const INSPECTION_QPD_PER_PROPERTY = 2e3;
+const INDEXING_NOT_FOUND_RE = /\b404\b|NOT_FOUND/i;
+const INSPECTION_HISTORY_LOOKBACK_MONTHS = 24;
+async function findLatestInspection(inspector, ctx, url) {
+	const now = /* @__PURE__ */ new Date();
+	const buckets = [];
+	for (let i = 0; i < INSPECTION_HISTORY_LOOKBACK_MONTHS; i++) {
+		const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1));
+		buckets.push(`${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`);
+	}
+	buckets.push("unknown");
+	for (const yearMonth of buckets) {
+		const matches = (await inspector.loadHistory(ctx, yearMonth))?.records.filter((r) => r.url === url) ?? [];
+		if (matches.length > 0) return matches.reduce((a, b) => a.inspectedAt >= b.inspectedAt ? a : b);
+	}
+}
+async function readUrlList(opts) {
+	if (opts.file) return (await readFile(opts.file, "utf8")).split("\n").map((l) => l.trim()).filter(Boolean);
+	const chunks = [];
+	for await (const chunk of process.stdin) chunks.push(chunk);
+	return Buffer.concat(chunks).toString("utf8").split("\n").map((l) => l.trim()).filter(Boolean);
+}
+const inspectSubCommand = defineCommand({
+	meta: {
+		name: "inspect",
+		description: "Run URL Inspection for a list of URLs and persist results to the local entity store"
+	},
+	args: {
+		site: {
+			type: "string",
+			alias: "s",
+			description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
+		},
+		file: {
+			type: "string",
+			alias: "f",
+			description: "Path to a file with one URL per line. If omitted, reads from stdin."
+		},
+		limit: {
+			type: "string",
+			description: `Max URLs to inspect this run (default: ${INSPECTION_QPD_PER_PROPERTY}, the per-property GSC daily quota)`
+		},
+		concurrency: {
+			type: "string",
+			alias: "c",
+			default: "4",
+			description: "Concurrent in-flight inspect calls (default: 4)"
+		},
+		...OUTPUT_ARGS
+	},
+	async run({ args }) {
+		const { json, quiet } = applyOutputMode(args);
+		const ctx = await createCommandContext({
+			needsAuth: true,
+			needsStore: true
+		});
+		const client = ctx.client;
+		const store = ctx.store;
+		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+		const limit = args.limit ? Number.parseInt(String(args.limit), 10) : INSPECTION_QPD_PER_PROPERTY;
+		const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
+		const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
+		if (urls.length === 0) {
+			logger.warn("No URLs to inspect.");
+			return;
+		}
+		if (urls.length === limit && limit < INSPECTION_QPD_PER_PROPERTY) logger.info(`Capping at --limit ${limit}`);
+		if (urls.length === INSPECTION_QPD_PER_PROPERTY) logger.info(`Hit per-property daily inspection quota (${INSPECTION_QPD_PER_PROPERTY}); remaining URLs will be queued for tomorrow.`);
+		const inspector = createInspectionStore({ dataSource: store.dataSource });
+		let completed = 0;
+		let failed = 0;
+		const records = [];
+		const failures = [];
+		await runWithConcurrency(urls, concurrency, async (url) => {
+			const result = await client.inspect(siteUrl, url).catch((err) => err);
+			if (result instanceof Error) {
+				failed++;
+				failures.push({
+					url,
+					error: result.message
+				});
+			} else {
+				const ix = result.inspectionResult;
+				const indexStatus = ix?.indexStatusResult;
+				records.push({
+					url,
+					inspectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+					indexStatus: indexStatus?.verdict ?? void 0,
+					lastCrawlTime: indexStatus?.lastCrawlTime ?? void 0,
+					googleCanonical: indexStatus?.googleCanonical ?? void 0,
+					userCanonical: indexStatus?.userCanonical ?? void 0,
+					coverageState: indexStatus?.coverageState ?? void 0,
+					robotsTxtState: indexStatus?.robotsTxtState ?? void 0,
+					indexingState: indexStatus?.indexingState ?? void 0,
+					pageFetchState: indexStatus?.pageFetchState ?? void 0,
+					mobileUsabilityVerdict: ix?.mobileUsabilityResult?.verdict ?? void 0,
+					richResultsVerdict: ix?.richResultsResult?.verdict ?? void 0,
+					raw: ix
+				});
+			}
+			completed++;
+			if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, `${url.slice(0, 60)}`)}`);
+		});
+		if (!quiet) process.stdout.write("\n");
+		await inspector.appendHistory({
+			userId: store.userId,
+			siteId: store.siteIdFor(siteUrl)
+		}, records);
+		if (json) console.log(JSON.stringify({
+			site: siteUrl,
+			inspected: records.length,
+			failed,
+			failures,
+			records
+		}, null, 2));
+		else if (!quiet) {
+			logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
+			if (failed > 0) {
+				logger.warn(`${failed} failed:`);
+				for (const f of failures.slice(0, 5)) console.log(`  ${f.url}: ${f.error}`);
+				if (failures.length > 5) console.log(`  ... and ${failures.length - 5} more`);
+			}
+		}
+		if (failed > 0) process.exit(1);
+	}
+});
+const showSubCommand = defineCommand({
+	meta: {
+		name: "show",
+		description: "Print the latest inspection record for a URL from the local entity store"
+	},
+	args: {
+		...OUTPUT_ARGS,
+		site: {
+			type: "string",
+			alias: "s",
+			description: "Site URL (defaults to config.defaultSite or prompt)"
+		},
+		url: {
+			type: "positional",
+			required: true,
+			description: "URL to look up"
+		}
+	},
+	async run({ args }) {
+		const { json } = applyOutputMode(args);
+		const ctx = await createCommandContext({
+			needsAuth: true,
+			needsStore: true
+		});
+		const store = ctx.store;
+		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+		const record = await findLatestInspection(createInspectionStore({ dataSource: store.dataSource }), {
+			userId: store.userId,
+			siteId: store.siteIdFor(siteUrl)
+		}, String(args.url));
+		if (!record) {
+			logger.warn(`No inspection record for ${args.url}`);
+			process.exit(1);
+		}
+		if (json) {
+			console.log(JSON.stringify(record, null, 2));
+			return;
+		}
+		console.log();
+		console.log(`  \x1B[1m${record.url}\x1B[0m`);
+		console.log(`  Inspected:    ${record.inspectedAt}`);
+		if (record.indexStatus) console.log(`  Index:        ${record.indexStatus}`);
+		if (record.lastCrawlTime) console.log(`  Last crawl:   ${record.lastCrawlTime}`);
+		if (record.googleCanonical) console.log(`  Canonical:    ${record.googleCanonical}`);
+		if (record.coverageState) console.log(`  Coverage:     ${record.coverageState}`);
+		if (record.mobileUsabilityVerdict) console.log(`  Mobile:       ${record.mobileUsabilityVerdict}`);
+		if (record.richResultsVerdict) console.log(`  Rich results: ${record.richResultsVerdict}`);
+		console.log();
+	}
+});
+const sitemapsSnapshotSubCommand = defineCommand({
+	meta: {
+		name: "snapshot",
+		description: "Fetch current sitemap state from GSC and persist to the local entity store"
+	},
+	args: {
+		...OUTPUT_ARGS,
+		site: {
+			type: "string",
+			alias: "s",
+			description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
+		}
+	},
+	async run({ args }) {
+		const { json, quiet } = applyOutputMode(args);
+		const ctx = await createCommandContext({
+			needsAuth: true,
+			needsStore: true
+		});
+		const client = ctx.client;
+		const store = ctx.store;
+		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+		const apiSitemaps = await client.sitemaps.list(siteUrl);
+		const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
+		const records = apiSitemaps.filter((s) => typeof s.path === "string").map((s) => ({
+			path: s.path,
+			capturedAt,
+			lastDownloaded: s.lastDownloaded ?? void 0,
+			lastSubmitted: s.lastSubmitted ?? void 0,
+			type: s.type ?? void 0,
+			isPending: s.isPending ?? void 0,
+			isSitemapsIndex: s.isSitemapsIndex ?? void 0,
+			errors: s.errors ?? void 0,
+			warnings: s.warnings ?? void 0,
+			contents: s.contents?.map((c) => ({
+				type: c.type ?? void 0,
+				submitted: c.submitted ?? void 0,
+				indexed: c.indexed ?? void 0
+			})),
+			raw: s
+		}));
+		await createSitemapStore({ dataSource: store.dataSource }).writeSnapshot({
+			userId: store.userId,
+			siteId: store.siteIdFor(siteUrl)
+		}, records);
+		if (json) {
+			console.log(JSON.stringify({
+				site: siteUrl,
+				capturedAt,
+				records
+			}, null, 2));
+			return;
+		}
+		if (!quiet) {
+			logger.success(`Captured ${records.length} sitemap(s) for ${siteUrl}`);
+			for (const r of records) {
+				const errors = r.errors && r.errors !== "0" ? ` \x1B[31merr=${r.errors}\x1B[0m` : "";
+				const warnings = r.warnings && r.warnings !== "0" ? ` \x1B[33mwarn=${r.warnings}\x1B[0m` : "";
+				const downloaded = r.lastDownloaded ? ` last=${r.lastDownloaded}` : "";
+				console.log(`  ${r.path}${downloaded}${errors}${warnings}`);
+			}
+		}
+	}
+});
+const sitemapsShowSubCommand = defineCommand({
+	meta: {
+		name: "show",
+		description: "Print the latest captured sitemap state for a feedpath"
+	},
+	args: {
+		...OUTPUT_ARGS,
+		site: {
+			type: "string",
+			alias: "s",
+			description: "Site URL (defaults to config.defaultSite or prompt)"
+		},
+		path: {
+			type: "positional",
+			required: true,
+			description: "Sitemap path (feedpath)"
+		}
+	},
+	async run({ args }) {
+		const { json } = applyOutputMode(args);
+		const ctx = await createCommandContext({
+			needsAuth: true,
+			needsStore: true
+		});
+		const store = ctx.store;
+		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+		const record = await createSitemapStore({ dataSource: store.dataSource }).getLatest({
+			userId: store.userId,
+			siteId: store.siteIdFor(siteUrl)
+		}, String(args.path));
+		if (!record) {
+			logger.warn(`No sitemap record for ${args.path}`);
+			process.exit(1);
+		}
+		if (json) {
+			console.log(JSON.stringify(record, null, 2));
+			return;
+		}
+		console.log();
+		console.log(`  \x1B[1m${record.path}\x1B[0m`);
+		console.log(`  Captured:     ${record.capturedAt}`);
+		if (record.lastDownloaded) console.log(`  Downloaded:   ${record.lastDownloaded}`);
+		if (record.lastSubmitted) console.log(`  Submitted:    ${record.lastSubmitted}`);
+		if (record.type) console.log(`  Type:         ${record.type}`);
+		if (record.errors) console.log(`  Errors:       ${record.errors}`);
+		if (record.warnings) console.log(`  Warnings:     ${record.warnings}`);
+		if (record.contents?.length) {
+			console.log(`  Contents:`);
+			for (const c of record.contents) {
+				const bits = [
+					c.type,
+					c.submitted && `submitted=${c.submitted}`,
+					c.indexed && `indexed=${c.indexed}`
+				].filter(Boolean).join("  ");
+				console.log(`    ${bits}`);
+			}
+		}
+		console.log();
+	}
+});
+const indexingSubCommand = defineCommand({
+	meta: {
+		name: "indexing",
+		description: "Snapshot Indexing API metadata per URL"
+	},
+	subCommands: { snapshot: defineCommand({
+		meta: {
+			name: "snapshot",
+			description: "Fetch Indexing API metadata (latest update/remove per URL) and persist to the local entity store"
+		},
+		args: {
+			site: {
+				type: "string",
+				alias: "s",
+				description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
+			},
+			file: {
+				type: "string",
+				alias: "f",
+				description: "Path to a file with one URL per line. If omitted, reads from stdin."
+			},
+			concurrency: {
+				type: "string",
+				alias: "c",
+				default: "4",
+				description: "Concurrent in-flight getMetadata calls (default: 4)"
+			},
+			...OUTPUT_ARGS
+		},
+		async run({ args }) {
+			const { quiet } = applyOutputMode(args);
+			const ctx = await createCommandContext({
+				needsAuth: true,
+				needsStore: true
+			});
+			const client = ctx.client;
+			const store = ctx.store;
+			const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+			const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
+			const urls = await readUrlList({ file: args.file ? String(args.file) : void 0 });
+			if (urls.length === 0) {
+				logger.warn("No URLs to fetch metadata for.");
+				return;
+			}
+			const records = [];
+			const failures = [];
+			let completed = 0;
+			await runWithConcurrency(urls, concurrency, async (url) => {
+				const result = await client.indexing.getMetadata(url).catch((err) => err);
+				if (result instanceof Error) if (INDEXING_NOT_FOUND_RE.test(result.message)) records.push({
+					url,
+					capturedAt: (/* @__PURE__ */ new Date()).toISOString()
+				});
+				else failures.push({
+					url,
+					error: result.message
+				});
+				else records.push({
+					url,
+					capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
+					latestUpdateAt: result.latestUpdate?.notifyTime ?? void 0,
+					latestRemoveAt: result.latestRemove?.notifyTime ?? void 0,
+					raw: result
+				});
+				completed++;
+				if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, url.slice(0, 60))}`);
+			});
+			if (!quiet) process.stdout.write("\n");
+			await createIndexingMetadataStore({ dataSource: store.dataSource }).writeBatch({
+				userId: store.userId,
+				siteId: store.siteIdFor(siteUrl)
+			}, records);
+			if (!quiet) {
+				logger.success(`Captured metadata for ${records.length}/${urls.length} URL(s)`);
+				if (failures.length > 0) {
+					logger.warn(`${failures.length} failed:`);
+					for (const f of failures.slice(0, 5)) console.log(`  ${f.url}: ${f.error}`);
+					if (failures.length > 5) console.log(`  ... and ${failures.length - 5} more`);
+				}
+			}
+			if (failures.length > 0) process.exit(1);
+		}
+	}) }
+});
+const entitiesCommand = defineCommand({
+	meta: {
+		name: "entities",
+		description: "Manage local entity snapshots (URL inspections, sitemaps, indexing metadata)"
+	},
+	subCommands: {
+		inspect: inspectSubCommand,
+		show: showSubCommand,
+		sitemaps: defineCommand({
+			meta: {
+				name: "sitemaps",
+				description: "Snapshot and inspect sitemap state per site"
+			},
+			subCommands: {
+				snapshot: sitemapsSnapshotSubCommand,
+				show: sitemapsShowSubCommand
+			}
+		}),
+		indexing: indexingSubCommand
+	}
+});
+export { entitiesCommand };

package/dist/_chunks/env-file.mjs

@@ -0,0 +1,53 @@
+import { useCliRuntime } from "./config.mjs";
+import process from "node:process";
+import fs from "node:fs";
+import path from "node:path";
+const ENV_LINE_RE = /^([^=]+)=(.*)$/;
+function parseEnvFile(envPath) {
+	let content;
+	try {
+		content = fs.readFileSync(envPath, "utf-8");
+	} catch {
+		return null;
+	}
+	const env = {};
+	for (const line of content.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed || trimmed.startsWith("#")) continue;
+		const match = trimmed.match(ENV_LINE_RE);
+		if (!match) continue;
+		const key = match[1].trim();
+		let value = match[2].trim();
+		if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
+		env[key] = value;
+	}
+	return env;
+}
+const appliedEnvKeys = /* @__PURE__ */ new Set();
+let loadedEnvPath = null;
+function getAppliedEnvKeys() {
+	return appliedEnvKeys;
+}
+function getLoadedEnvPath() {
+	return loadedEnvPath;
+}
+function loadEnvFromCwd() {
+	return loadEnvFile({
+		cwd: process.cwd(),
+		env: useCliRuntime().environment
+	});
+}
+function loadEnvFile(opts) {
+	const envPath = path.join(opts.cwd, ".env");
+	const parsed = parseEnvFile(envPath);
+	if (!parsed) return [];
+	loadedEnvPath = envPath;
+	const applied = [];
+	for (const [key, value] of Object.entries(parsed)) if (opts.env[key] === void 0) {
+		opts.env[key] = value;
+		applied.push(key);
+		appliedEnvKeys.add(key);
+	}
+	return applied;
+}
+export { getAppliedEnvKeys, getLoadedEnvPath, loadEnvFromCwd, parseEnvFile };

package/dist/_chunks/environment.mjs

@@ -0,0 +1,30 @@
+import { useCliRuntime } from "./config.mjs";
+function resolveCliEnvironment(values = useCliRuntime().environment) {
+	return {
+		accessToken: values.GSC_ACCESS_TOKEN || values.GOOGLE_ACCESS_TOKEN,
+		clientId: values.GSC_CLIENT_ID || values.GOOGLE_CLIENT_ID,
+		clientSecret: values.GSC_CLIENT_SECRET || values.GOOGLE_CLIENT_SECRET,
+		configDir: values.GSCDUMP_CONFIG_DIR,
+		forceColor: Boolean(values.FORCE_COLOR),
+		noColor: Boolean(values.NO_COLOR),
+		profile: values.GSCDUMP_PROFILE,
+		refreshToken: values.GSC_REFRESH_TOKEN || values.GOOGLE_REFRESH_TOKEN,
+		serviceAccountPath: values.GSC_SERVICE_ACCOUNT_JSON || values.GOOGLE_APPLICATION_CREDENTIALS,
+		values
+	};
+}
+function pickCliEnvironmentValue(names, values = useCliRuntime().environment) {
+	for (const envVar of names) {
+		const value = values[envVar];
+		if (value) return {
+			envVar,
+			value
+		};
+	}
+	return null;
+}
+function applyCliEnvironment(updates, values = useCliRuntime().environment) {
+	for (const [key, value] of Object.entries(updates)) if (value === void 0) delete values[key];
+	else values[key] = value;
+}
+export { applyCliEnvironment, pickCliEnvironmentValue, resolveCliEnvironment };

package/dist/_chunks/error-handler.mjs

@@ -0,0 +1,86 @@
+import { formatAuthProvenance, isAuthError } from "./auth.mjs";
+import process from "node:process";
+import { isQueryError } from "gscdump/query";
+import { formatErrorForCli } from "gscdump/api";
+import { isAnalysisError } from "@gscdump/analysis/errors";
+import { isEngineError } from "@gscdump/engine/errors";
+function extractQueryError(error) {
+	if (isQueryError(error)) return error;
+	const tagged = error.queryError;
+	return isQueryError(tagged) ? tagged : null;
+}
+function extractEngineError(error) {
+	if (isEngineError(error)) return error;
+	const tagged = error.engineError;
+	return isEngineError(tagged) ? tagged : null;
+}
+function extractAnalysisError(error) {
+	if (isAnalysisError(error)) return error;
+	const tagged = error.analysisError;
+	return isAnalysisError(tagged) ? tagged : null;
+}
+function suggestionForTypedError(error) {
+	const query = extractQueryError(error);
+	if (query) switch (query.kind) {
+		case "unresolvable-dataset": return "This breakdown spans separate stored tables. Re-run with --live to compute it from the GSC API.";
+		case "unsupported-capability": return `The local engine lacks the "${query.capability}" capability for ${query.context}. Re-run with --live.`;
+		case "missing-date-range": return "Add a date range, e.g. --start YYYY-MM-DD --end YYYY-MM-DD.";
+		case "invalid-row-limit":
+		case "invalid-start-row":
+		case "invalid-data-state":
+		case "invalid-aggregation-type":
+		case "invalid-builder-state": return "";
+	}
+	const analysis = extractAnalysisError(error);
+	if (analysis) switch (analysis.kind) {
+		case "missing-report-param": return `Report "${analysis.report}" needs --${analysis.param}.`;
+		case "missing-comparison-window": return `Report "${analysis.report}" is a comparison report; pass --vs prev-period (or --vs yoy).`;
+		case "missing-brand-terms": return "Pass --brand-terms to segment branded vs non-branded queries.";
+		case "unknown-report": return `Unknown report. Available: ${analysis.available.join(", ")}.`;
+		case "unknown-analyzer": return "Run `gscdump report list` to see available reports/analyzers.";
+		case "required-step-failed": return suggestionForTypedError(analysis.cause);
+	}
+	const engine = extractEngineError(error);
+	if (engine) switch (engine.kind) {
+		case "attached-table-missing": return `Local store is missing table(s): ${engine.missing.join(", ")}. Run \`gscdump sync\` first, or pass --live.`;
+		case "analyzer-not-found":
+		case "analyzer-capability-missing": return "This analysis is not available for the chosen source. Re-run with --live.";
+		case "invalid-year-month":
+		case "invalid-snapshot-filename":
+		case "invalid-schema-identifier": return "The local store appears corrupt. Re-run `gscdump sync` to rebuild it.";
+		default: return "";
+	}
+	return "";
+}
+var LocalStoreUnsupportedError = class extends Error {
+	tool;
+	mode;
+	constructor(tool, mode) {
+		super(`analysis "${tool}" has no implementation for the ${mode} source`);
+		this.name = "LocalStoreUnsupportedError";
+		this.tool = tool;
+		this.mode = mode;
+	}
+};
+async function gscErrorHandler(error) {
+	console.error();
+	if (error instanceof LocalStoreUnsupportedError) {
+		console.error(formatErrorForCli(error));
+		if (error.mode === "local") console.error("Pass --live to run against the GSC API.");
+		console.error();
+		process.exit(1);
+	}
+	console.error(formatErrorForCli(error));
+	const typedHint = suggestionForTypedError(error);
+	if (typedHint) {
+		console.error();
+		console.error(typedHint);
+	}
+	if (isAuthError(error)) {
+		console.error();
+		console.error(await formatAuthProvenance());
+	}
+	console.error();
+	process.exit(1);
+}
+export { LocalStoreUnsupportedError, gscErrorHandler };