@gscdump/cli 0.37.3 → 0.37.5

package/dist/_chunks/indexing.mjs

@@ -0,0 +1,314 @@
+import { OUTPUT_ARGS, applyOutputMode, logger, readUrlList } from "./utils.mjs";
+import { createCommandContext } from "./context.mjs";
+import { gscErrorHandler } from "./error-handler.mjs";
+import process from "node:process";
+import { defineCommand } from "citty";
+import { batchRequestIndexing, fetchSitemapUrls, getIndexingMetadata, requestIndexing, runSequentialBatch } from "gscdump/api";
+const RETRIES_ARG = { retries: {
+	type: "string",
+	description: "Override per-call retry count (default: 3)"
+} };
+function parseRetries(v) {
+	if (v == null || v === "") return void 0;
+	const n = Number.parseInt(String(v), 10);
+	return Number.isFinite(n) && n >= 0 ? n : void 0;
+}
+async function resolveUrlSource(args) {
+	const fromSitemap = args["from-sitemap"];
+	if (fromSitemap) return fetchSitemapUrls(String(fromSitemap)).catch((e) => {
+		logger.error(`Sitemap fetch failed: ${e.message}`);
+		process.exit(1);
+	});
+	return readUrlList(args);
+}
+const submitCommand = defineCommand({
+	meta: {
+		name: "submit",
+		description: "Notify Google of a new or updated URL (URL_UPDATED)"
+	},
+	args: {
+		url: {
+			type: "positional",
+			required: true,
+			description: "URL to submit"
+		},
+		...OUTPUT_ARGS,
+		...RETRIES_ARG
+	},
+	async run({ args }) {
+		applyOutputMode(args);
+		const result = await requestIndexing((await createCommandContext({
+			needsAuth: true,
+			fetchOptions: { retry: parseRetries(args.retries) }
+		})).client, args.url, { type: "URL_UPDATED" }).catch(gscErrorHandler);
+		if (args.json) {
+			console.log(JSON.stringify(result, null, 2));
+			return;
+		}
+		logger.success(`Submitted: ${result.url}`);
+		if (result.notifyTime) console.log(`  Notified: ${result.notifyTime}`);
+	}
+});
+const removeCommand = defineCommand({
+	meta: {
+		name: "remove",
+		description: "Notify Google a URL has been removed (URL_DELETED)"
+	},
+	args: {
+		url: {
+			type: "positional",
+			required: true,
+			description: "URL to mark removed"
+		},
+		...OUTPUT_ARGS,
+		...RETRIES_ARG
+	},
+	async run({ args }) {
+		applyOutputMode(args);
+		const result = await requestIndexing((await createCommandContext({
+			needsAuth: true,
+			fetchOptions: { retry: parseRetries(args.retries) }
+		})).client, args.url, { type: "URL_DELETED" }).catch(gscErrorHandler);
+		if (args.json) {
+			console.log(JSON.stringify(result, null, 2));
+			return;
+		}
+		logger.success(`Removed: ${result.url}`);
+		if (result.notifyTime) console.log(`  Notified: ${result.notifyTime}`);
+	}
+});
+const statusCommand = defineCommand({
+	meta: {
+		name: "status",
+		description: "Show indexing notification metadata for a URL"
+	},
+	args: {
+		url: {
+			type: "positional",
+			required: true,
+			description: "URL to inspect"
+		},
+		json: {
+			type: "boolean",
+			default: false,
+			description: "Output as JSON"
+		},
+		quiet: {
+			type: "boolean",
+			alias: "q",
+			default: false,
+			description: "Suppress info/success output"
+		}
+	},
+	async run({ args }) {
+		applyOutputMode(args);
+		const meta = await getIndexingMetadata((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
+		if (args.json) {
+			console.log(JSON.stringify(meta, null, 2));
+			return;
+		}
+		const fmtNotification = (n) => {
+			if (!n?.notifyTime) return "never";
+			return n.type ? `${n.notifyTime} (${n.type})` : n.notifyTime;
+		};
+		const update = meta.latestUpdate;
+		const remove = meta.latestRemove;
+		console.log();
+		console.log(`  \x1B[1mURL:\x1B[0m ${meta.url}`);
+		console.log(`  Last update notify: ${fmtNotification(update)}`);
+		console.log(`  Last remove notify: ${fmtNotification(remove)}`);
+		console.log();
+	}
+});
+const INDEXING_DAILY_QUOTA = 200;
+const INDEXING_PER_MINUTE_QUOTA = 600;
+const quotaCommand = defineCommand({
+	meta: {
+		name: "quota",
+		description: "Show documented Indexing API quotas (no live counters; quota usage is not exposed by the API)"
+	},
+	args: { ...OUTPUT_ARGS },
+	async run({ args }) {
+		const { json } = applyOutputMode(args);
+		const payload = {
+			perDay: INDEXING_DAILY_QUOTA,
+			perMinute: INDEXING_PER_MINUTE_QUOTA,
+			note: "Documented defaults. Google does not expose live counters; track yours by counting submit calls.",
+			docs: "https://developers.google.com/search/apis/indexing-api/v3/quota-pricing"
+		};
+		if (json) {
+			console.log(JSON.stringify(payload, null, 2));
+			return;
+		}
+		console.log();
+		console.log(`  \x1B[1mIndexing API quota\x1B[0m`);
+		console.log(`    Per day:    ${payload.perDay}`);
+		console.log(`    Per minute: ${payload.perMinute}`);
+		console.log();
+		console.log(`  \x1B[90m${payload.note}\x1B[0m`);
+		console.log(`  \x1B[90mDocs: ${payload.docs}\x1B[0m`);
+		console.log();
+	}
+});
+const indexingCommand = defineCommand({
+	meta: {
+		name: "indexing",
+		description: "Notify Google about URL updates/removals (Indexing API)"
+	},
+	subCommands: {
+		"submit": submitCommand,
+		"remove": removeCommand,
+		"status": statusCommand,
+		"batch": defineCommand({
+			meta: {
+				name: "batch",
+				description: "Submit many URLs from a file or stdin (one URL per line)"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				"urls": {
+					type: "positional",
+					required: false,
+					description: "URLs (or use --file/--from-sitemap/stdin)"
+				},
+				"file": {
+					type: "string",
+					alias: "f",
+					description: "File with URLs (one per line)"
+				},
+				"from-sitemap": {
+					type: "string",
+					description: "Sitemap URL (or sitemap index) to pull URLs from"
+				},
+				"type": {
+					type: "string",
+					default: "URL_UPDATED",
+					description: "URL_UPDATED or URL_DELETED"
+				},
+				"delay-ms": {
+					type: "string",
+					default: "100",
+					description: "Delay between requests"
+				},
+				"concurrency": {
+					type: "string",
+					alias: "c",
+					default: "1",
+					description: "Concurrent in-flight requests"
+				},
+				"yes": {
+					type: "boolean",
+					alias: "y",
+					default: false,
+					description: "Skip the over-quota confirmation prompt"
+				},
+				"retries": {
+					type: "string",
+					description: "Override per-call retry count (default: 3)"
+				}
+			},
+			async run({ args }) {
+				applyOutputMode(args);
+				const urls = await resolveUrlSource(args);
+				if (urls.length === 0) {
+					logger.error("No URLs provided. Pass URLs as args, --file, --from-sitemap, or stdin.");
+					process.exit(1);
+				}
+				const type = String(args.type);
+				if (type !== "URL_UPDATED" && type !== "URL_DELETED") {
+					logger.error(`Invalid --type: ${type}. Use URL_UPDATED or URL_DELETED.`);
+					process.exit(1);
+				}
+				if (urls.length > INDEXING_DAILY_QUOTA && !args.yes && !args.json) {
+					logger.warn(`Submitting ${urls.length} URLs but the Indexing API daily quota is ${INDEXING_DAILY_QUOTA}/day.`);
+					logger.warn(`Excess submissions will fail with quota errors. Pass --yes to proceed anyway.`);
+					process.exit(1);
+				}
+				if (urls.length > INDEXING_DAILY_QUOTA && !args.json && !args.quiet) logger.warn(`Proceeding with ${urls.length} URLs (over the ${INDEXING_DAILY_QUOTA}/day quota). Excess will fail.`);
+				const ctx = await createCommandContext({
+					needsAuth: true,
+					fetchOptions: { retry: parseRetries(args.retries) }
+				});
+				const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
+				const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
+				if (!args.json && !args.quiet) logger.info(`Submitting ${urls.length} URLs (${type}) ...`);
+				const results = await batchRequestIndexing(ctx.client, urls, {
+					type,
+					delayMs,
+					concurrency,
+					onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
+				}).catch(gscErrorHandler);
+				if (args.json) {
+					console.log(JSON.stringify(results, null, 2));
+					return;
+				}
+				if (!args.quiet) logger.success(`Submitted ${results.length}/${urls.length} URLs`);
+			}
+		}),
+		"batch-status": defineCommand({
+			meta: {
+				name: "batch-status",
+				description: "Get indexing notification metadata for many URLs"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				"urls": {
+					type: "positional",
+					required: false,
+					description: "URLs (or use --file/--from-sitemap/stdin)"
+				},
+				"file": {
+					type: "string",
+					alias: "f",
+					description: "File with URLs (one per line)"
+				},
+				"from-sitemap": {
+					type: "string",
+					description: "Sitemap URL (or sitemap index) to pull URLs from"
+				},
+				"delay-ms": {
+					type: "string",
+					default: "100",
+					description: "Delay between requests"
+				},
+				"concurrency": {
+					type: "string",
+					alias: "c",
+					default: "1",
+					description: "Concurrent in-flight requests"
+				},
+				"retries": {
+					type: "string",
+					description: "Override per-call retry count (default: 3)"
+				}
+			},
+			async run({ args }) {
+				applyOutputMode(args);
+				const urls = await resolveUrlSource(args);
+				if (urls.length === 0) {
+					logger.error("No URLs provided. Pass URLs as args, --file, --from-sitemap, or stdin.");
+					process.exit(1);
+				}
+				const ctx = await createCommandContext({
+					needsAuth: true,
+					fetchOptions: { retry: parseRetries(args.retries) }
+				});
+				const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
+				const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
+				if (!args.json && !args.quiet) logger.info(`Fetching status for ${urls.length} URLs ...`);
+				const results = await runSequentialBatch(urls, (url) => getIndexingMetadata(ctx.client, url), {
+					delayMs,
+					concurrency,
+					onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
+				}).catch(gscErrorHandler);
+				if (args.json) {
+					console.log(JSON.stringify(results, null, 2));
+					return;
+				}
+				if (!args.quiet) logger.success(`Fetched ${results.length}/${urls.length} URLs`);
+			}
+		}),
+		"quota": quotaCommand
+	}
+});
+export { indexingCommand };

package/dist/_chunks/init.mjs

@@ -0,0 +1,190 @@
+import { defaultDataDir, loadConfig, saveConfig } from "./config.mjs";
+import { applyCliEnvironment } from "./environment.mjs";
+import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "./utils.mjs";
+import { authenticate, getAuthCredentials, loadTokens, resolveBYOK, saveTokens } from "./auth.mjs";
+import process from "node:process";
+import { defineCommand } from "citty";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { confirm, isCancel, text } from "@clack/prompts";
+import { googleSearchConsole } from "gscdump/api";
+const ENV_LINE_RE = /^([^=]+)=(.*)$/;
+async function promptDataDir(existing) {
+	const fallback = existing ?? defaultDataDir();
+	const answer = await text({
+		message: "Where should Parquet data be stored?",
+		placeholder: fallback,
+		defaultValue: fallback
+	});
+	if (isCancel(answer)) process.exit(1);
+	return String(answer) || fallback;
+}
+async function loadEnvFile() {
+	const envPath = path.join(process.cwd(), ".env");
+	const content = await fs.readFile(envPath, "utf-8").catch(() => null);
+	if (!content) return null;
+	const env = {};
+	for (const line of content.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed || trimmed.startsWith("#")) continue;
+		const match = trimmed.match(ENV_LINE_RE);
+		if (match) {
+			const key = match[1].trim();
+			let value = match[2].trim();
+			if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
+			env[key] = value;
+		}
+	}
+	return env;
+}
+const initCommand = defineCommand({
+	meta: {
+		name: "init",
+		description: "Set up GSCDump authentication"
+	},
+	args: {
+		"force": {
+			type: "boolean",
+			alias: "f",
+			description: "Force re-initialization"
+		},
+		"no-store": {
+			type: "boolean",
+			default: false,
+			description: "Skip dataDir prompt (auth-only setup)"
+		},
+		...OUTPUT_ARGS
+	},
+	async run({ args }) {
+		applyOutputMode(args);
+		const config = await loadConfig();
+		if (config.clientId && config.clientSecret && !args.force) {
+			logger.info("Already configured");
+			const tokens = await loadTokens();
+			if (!tokens) logger.info("No saved tokens. Run `gscdump auth login` to authenticate.");
+			else {
+				const isExpired = tokens.expiry_date && tokens.expiry_date < Date.now();
+				if (isExpired && tokens.refresh_token) logger.info("Tokens expired but refresh available. Any live command will auto-refresh, or run `gscdump auth refresh`.");
+				else if (isExpired) logger.warn("Tokens expired without a refresh token. Run `gscdump auth login` to re-authenticate.");
+				else logger.info("Next: `gscdump sites` to list properties, or `gscdump query --help`.");
+			}
+			logger.info("Run `gscdump init --force` to reconfigure.");
+			return;
+		}
+		const byok = resolveBYOK();
+		if (byok) {
+			const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
+			await saveConfig({
+				...config,
+				dataDir: dataDir ?? config.dataDir
+			});
+			logger.success(`BYOK detected (${typeof byok === "string" ? "access-token" : "refresh-token"}) — auth setup skipped`);
+			logger.success("Setup complete! Run gscdump to get started.");
+			return;
+		}
+		const envFile = await loadEnvFile();
+		const envCid = envFile?.GSC_CLIENT_ID ?? envFile?.GOOGLE_CLIENT_ID;
+		const envSec = envFile?.GSC_CLIENT_SECRET ?? envFile?.GOOGLE_CLIENT_SECRET;
+		const envRef = envFile?.GSC_REFRESH_TOKEN ?? envFile?.GOOGLE_REFRESH_TOKEN;
+		const envAcc = envFile?.GSC_ACCESS_TOKEN ?? envFile?.GOOGLE_ACCESS_TOKEN;
+		if (envCid && envSec && envRef) {
+			logger.info("Found .env file with credentials");
+			applyCliEnvironment({
+				GOOGLE_ACCESS_TOKEN: envAcc,
+				GOOGLE_CLIENT_ID: envCid,
+				GOOGLE_CLIENT_SECRET: envSec,
+				GOOGLE_REFRESH_TOKEN: envRef
+			});
+			await saveConfig({
+				...config,
+				clientId: envCid,
+				clientSecret: envSec,
+				dataDir: config.dataDir ?? defaultDataDir()
+			});
+			const creds = (await authenticate({
+				clientId: envCid,
+				clientSecret: envSec
+			}, false)).credentials;
+			if (creds.access_token) await saveTokens({
+				access_token: creds.access_token,
+				refresh_token: creds.refresh_token || envRef,
+				expiry_date: creds.expiry_date
+			});
+			console.log();
+			logger.success("Setup complete using .env credentials! Run gscdump to get started.");
+			return;
+		}
+		console.log();
+		console.log("  \x1B[1mWelcome to GSCDump!\x1B[0m");
+		console.log("  \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
+		console.log();
+		const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
+		const credentials = await getAuthCredentials(true);
+		await saveConfig({
+			...config,
+			...dataDir ? { dataDir } : {},
+			clientId: credentials.clientId,
+			clientSecret: credentials.clientSecret
+		});
+		await smokeTest(await authenticate(credentials, true));
+		await maybeWriteEnvFile(credentials.clientId, credentials.clientSecret);
+		console.log();
+		logger.success("Setup complete! Run gscdump to get started.");
+	}
+});
+async function smokeTest(oauth) {
+	await runSmokeTest(oauth);
+}
+async function runSmokeTest(oauth) {
+	const sites = await googleSearchConsole(oauth).sites().catch((e) => e);
+	if (sites instanceof Error) {
+		const msg = sites.message;
+		const apiDisabledMatch = msg.match(/projects?\/(\d+)/) ?? msg.match(/project (\d+)/);
+		if (/has not been used in project|API has not been used|SERVICE_DISABLED/i.test(msg)) {
+			logger.error("Search Console API is not enabled for this Google Cloud project.");
+			const project = apiDisabledMatch?.[1];
+			const url = project ? `https://console.developers.google.com/apis/api/searchconsole.googleapis.com/overview?project=${project}` : "https://console.developers.google.com/apis/api/searchconsole.googleapis.com/overview";
+			logger.info(`Enable it here, then retry: ${url}`);
+			logger.info("Note: it can take a few minutes to propagate after enabling.");
+			return;
+		}
+		if (/insufficient|scope|forbidden|403/i.test(msg)) {
+			logger.warn(`Smoke test failed (likely missing scopes): ${msg}`);
+			logger.info("Run `gscdump auth login --force` to re-consent with the required scopes.");
+			return;
+		}
+		logger.warn(`Smoke test failed: ${msg}`);
+		logger.info("Auth saved, but verify via `gscdump auth status` / `gscdump doctor`.");
+		return;
+	}
+	logger.success(`Verified: ${sites.length} GSC site(s) accessible`);
+}
+async function maybeWriteEnvFile(clientId, clientSecret) {
+	const tokens = await loadTokens();
+	if (!tokens?.refresh_token) return;
+	const wants = await confirm({
+		message: "Write a `.env` file with these credentials? (handy for CI / other machines)",
+		initialValue: false
+	});
+	if (isCancel(wants) || !wants) return;
+	const envPath = path.join(process.cwd(), ".env");
+	if (await fs.stat(envPath).then(() => true).catch(() => false)) {
+		const overwrite = await confirm({
+			message: `${displayPath(envPath)} exists. Overwrite?`,
+			initialValue: false
+		});
+		if (isCancel(overwrite) || !overwrite) {
+			logger.info(`Skipped — keep credentials at ${displayPath(envPath)} manually if needed`);
+			return;
+		}
+	}
+	const content = [
+		`GSC_CLIENT_ID=${clientId}`,
+		`GSC_CLIENT_SECRET=${clientSecret}`,
+		`GSC_REFRESH_TOKEN=${tokens.refresh_token}`,
+		""
+	].join("\n");
+	await fs.writeFile(envPath, content, { mode: 384 });
+	logger.success(`Wrote ${displayPath(envPath)}`);
+}
+export { initCommand, runSmokeTest };