@gscdump/cli 3.8.0 → 4.0.0

package/dist/commands/indexing-urls.mjs

@@ -0,0 +1,185 @@
+import { terminalOutputOptions } from "../render/terminal.mjs";
+import { applyOutputMode, logger, parseIntegerOption, toCSV } from "../utils.mjs";
+import { renderTable } from "../render/layout.mjs";
+import { HOSTED_ARGS, resolveHostedSite } from "../hosted-site.mjs";
+import { defineCommand } from "citty";
+const STATUSES = [
+	"indexed",
+	"not_indexed",
+	"pending"
+];
+const FORMATS = [
+	"table",
+	"json",
+	"csv"
+];
+const MAX_PAGE_SIZE = 500;
+const CSV_COLUMNS = [
+	"url",
+	"verdict",
+	"coverageState",
+	"issueType",
+	"lastCrawlTime",
+	"lastCheckedAt",
+	"googleCanonical",
+	"userCanonical",
+	"sitemaps"
+];
+function parseChoice(value, flag, allowed) {
+	if (value === void 0 || value === null || value === "") return void 0;
+	if (!allowed.includes(String(value))) throw new Error(`Invalid ${flag}: ${String(value)}. Use ${allowed.join(", ")}.`);
+	return String(value);
+}
+async function fetchAllPages(client, site, query) {
+	const urls = [];
+	let first;
+	while (true) {
+		const { data } = await client.listSiteIndexingUrls({
+			params: { siteId: site.siteId },
+			query: {
+				...query,
+				limit: MAX_PAGE_SIZE,
+				offset: urls.length
+			}
+		});
+		first ??= data;
+		if (data.pagination.offset !== urls.length || data.pagination.hasMore && data.urls.length === 0) throw new Error("The hosted API returned inconsistent indexing pagination. Run the command again.");
+		urls.push(...data.urls);
+		if (!data.pagination.hasMore) return {
+			...first,
+			urls,
+			pagination: {
+				...data.pagination,
+				total: urls.length,
+				limit: urls.length,
+				offset: 0,
+				hasMore: false
+			}
+		};
+	}
+}
+function csvRow(row) {
+	return {
+		...row,
+		sitemaps: (row.sitemaps ?? []).join(" ")
+	};
+}
+const indexingUrlsCommand = defineCommand({
+	meta: {
+		name: "urls",
+		description: "List Google URL Inspection results for a Site's URLs, filtered by status (hosted)"
+	},
+	args: {
+		...HOSTED_ARGS,
+		status: {
+			type: "string",
+			description: `Filter by status: ${STATUSES.join(", ")}`
+		},
+		search: {
+			type: "string",
+			description: "Only URLs that contain this text"
+		},
+		limit: {
+			type: "string",
+			alias: "l",
+			default: "100",
+			description: `Rows per page, maximum ${MAX_PAGE_SIZE}`
+		},
+		offset: {
+			type: "string",
+			default: "0",
+			description: "Rows to skip before the page"
+		},
+		all: {
+			type: "boolean",
+			default: false,
+			description: "Read every page; ignores --limit and --offset"
+		},
+		format: {
+			type: "string",
+			alias: "f",
+			default: "table",
+			description: `Output format: ${FORMATS.join(", ")}`
+		},
+		json: {
+			type: "boolean",
+			default: false,
+			description: "Output as JSON (same as --format json)"
+		},
+		quiet: {
+			type: "boolean",
+			alias: "q",
+			default: false,
+			description: "Suppress info output"
+		}
+	},
+	async run({ args }) {
+		const format = args.json ? "json" : parseChoice(args.format, "--format", FORMATS) ?? "table";
+		const status = parseChoice(args.status, "--status", STATUSES);
+		const limit = parseIntegerOption(args.limit, "--limit") ?? 100;
+		const offset = parseIntegerOption(args.offset, "--offset", 0) ?? 0;
+		if (limit > MAX_PAGE_SIZE) throw new Error(`--limit must be ${MAX_PAGE_SIZE} or less. Use --all to read every page.`);
+		const search = args.search ? String(args.search) : void 0;
+		applyOutputMode({
+			json: format !== "table",
+			quiet: args.quiet
+		});
+		const filters = {
+			...status ? { status } : {},
+			...search ? { search } : {}
+		};
+		const { client, site } = await resolveHostedSite(args, {
+			name: "indexing urls",
+			localAlternative: "pipe `gscdump sitemaps urls <sitemap-url>` into `gscdump inspect --site <site>`"
+		});
+		const page = args.all ? await fetchAllPages(client, site, filters) : (await client.listSiteIndexingUrls({
+			params: { siteId: site.siteId },
+			query: {
+				...filters,
+				limit,
+				offset
+			}
+		})).data;
+		if (format === "json") {
+			console.log(JSON.stringify(page, null, 2));
+			return;
+		}
+		if (format === "csv") {
+			console.log(toCSV(page.urls.map(csvRow), [...CSV_COLUMNS]));
+			return;
+		}
+		const options = terminalOutputOptions();
+		if (page.urls.length === 0) {
+			logger.info(`No URLs${status ? ` with status ${status}` : ""} for ${site.siteUrl}.`);
+			return;
+		}
+		for (const line of renderTable(page.urls.map((row) => ({
+			url: row.url,
+			coverage: row.coverageState ?? row.verdict ?? "unknown",
+			lastCrawl: row.lastCrawlTime?.slice(0, 10) ?? "never",
+			sitemaps: row.sitemaps?.length ?? 0
+		})), [
+			{
+				key: "url",
+				label: "URL"
+			},
+			{
+				key: "coverage",
+				label: "Coverage"
+			},
+			{
+				key: "lastCrawl",
+				label: "Last crawl"
+			},
+			{
+				key: "sitemaps",
+				label: "Sitemaps",
+				numeric: true
+			}
+		], options)) console.log(line);
+		const { total, offset: start, hasMore } = page.pagination;
+		logger.info(`Showing ${start + 1}-${start + page.urls.length} of ${total} URLs for ${site.siteUrl}.`);
+		if (hasMore) logger.info(`For the next page, pass --offset ${start + page.urls.length}. To read every page, pass --all.`);
+	}
+});
+export { indexingUrlsCommand };

package/dist/commands/indexing.mjs

@@ -1,8 +1,8 @@
 import { indexingCommandMeta } from "../command-meta.mjs";
 import { OUTPUT_ARGS, applyOutputMode, logger, parseIntegerOption, readUrlList } from "../utils.mjs";
 import { createCommandContext } from "../context.mjs";
-import { gscErrorHandler } from "../error-handler.mjs";
 import { loadSitemapUrls } from "../sitemap.mjs";
+import { indexingUrlsCommand } from "./indexing-urls.mjs";
 import process from "node:process";
 import { defineCommand } from "citty";
 import { batchRequestIndexing, getIndexingMetadata, requestIndexing, runSequentialBatch } from "gscdump/indexing";
@@ -21,7 +21,10 @@ async function resolveUrlSource(args) {
 		if (!result.value.complete) logger.warn("Sitemap walk was incomplete; indexing only the URLs that were read");
 		return result.value.urls;
 	}
-	return readUrlList(args);
+	return readUrlList({
+		file: args.file,
+		positionals: args._
+	});
 }
 const submitCommand = defineCommand({
 	meta: {
@@ -43,7 +46,7 @@ const submitCommand = defineCommand({
 			needsAuth: true,
 			fetchOptions: { retry: parseIntegerOption(args.retries, "--retries", 0) }
 		});
-		const result = await requestIndexing(ctx.client, args.url, { type: "URL_UPDATED" }).catch(gscErrorHandler);
+		const result = await requestIndexing(ctx.client, args.url, { type: "URL_UPDATED" });
 		if (args.json) {
 			console.log(JSON.stringify(result, null, 2));
 			return;
@@ -72,7 +75,7 @@ const removeCommand = defineCommand({
 			needsAuth: true,
 			fetchOptions: { retry: parseIntegerOption(args.retries, "--retries", 0) }
 		});
-		const result = await requestIndexing(ctx.client, args.url, { type: "URL_DELETED" }).catch(gscErrorHandler);
+		const result = await requestIndexing(ctx.client, args.url, { type: "URL_DELETED" });
 		if (args.json) {
 			console.log(JSON.stringify(result, null, 2));
 			return;
@@ -107,7 +110,7 @@ const statusCommand = defineCommand({
 	async run({ args }) {
 		applyOutputMode(args);
 		const ctx = await createCommandContext({ needsAuth: true });
-		const meta = await getIndexingMetadata(ctx.client, args.url).catch(gscErrorHandler);
+		const meta = await getIndexingMetadata(ctx.client, args.url);
 		if (args.json) {
 			console.log(JSON.stringify(meta, null, 2));
 			return;
@@ -234,7 +237,7 @@ const batchCommand = defineCommand({
 			delayMs,
 			concurrency,
 			onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
-		}).catch(gscErrorHandler);
+		});
 		if (args.json) {
 			console.log(JSON.stringify(results, null, 2));
 			return;
@@ -298,7 +301,7 @@ const batchStatusCommand = defineCommand({
 			delayMs,
 			concurrency,
 			onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
-		}).catch(gscErrorHandler);
+		});
 		if (args.json) {
 			console.log(JSON.stringify(results, null, 2));
 			return;
@@ -314,7 +317,8 @@ const indexingCommand = defineCommand({
 		"status": statusCommand,
 		"batch": batchCommand,
 		"batch-status": batchStatusCommand,
-		"quota": quotaCommand
+		"quota": quotaCommand,
+		"urls": indexingUrlsCommand
 	}
 });
 export { indexingCommand };

package/dist/commands/init.mjs

@@ -1,8 +1,11 @@
 import { initCommandMeta } from "../command-meta.mjs";
+import { useCliRuntime } from "../runtime.mjs";
+import { saveAuthentication } from "../auth-state.mjs";
 import { defaultDataDir, loadConfig, saveConfig } from "../config.mjs";
 import { applyCliEnvironment } from "../environment.mjs";
 import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "../utils.mjs";
 import { authenticate, getAuth, loadTokens, resolveBYOK, saveTokens } from "../auth.mjs";
+import { loginCloud } from "./auth.mjs";
 import process from "node:process";
 import { defineCommand } from "citty";
 import fs from "node:fs/promises";
@@ -10,8 +13,12 @@ import path from "node:path";
 import { confirm, isCancel, text } from "@clack/prompts";
 import { googleSearchConsole } from "gscdump/client";
 const ENV_LINE_RE = /^([^=]+)=(.*)$/;
+function canPrompt() {
+	return process.stdin.isTTY === true;
+}
 async function promptDataDir(existing) {
 	const fallback = existing ?? defaultDataDir();
+	if (!canPrompt()) return fallback;
 	const answer = await text({
 		message: "Where should Parquet data be stored?",
 		placeholder: fallback,
@@ -46,16 +53,35 @@ const initCommand = defineCommand({
 			alias: "f",
 			description: "Force re-initialization"
 		},
-		"no-store": {
+		"store": {
 			type: "boolean",
-			default: false,
-			description: "Skip dataDir prompt (auth-only setup)"
+			default: true,
+			description: "Ask where to keep the local Store",
+			negativeDescription: "Skip the Store location prompt (authentication only)"
+		},
+		"mode": {
+			type: "string",
+			description: "Authentication mode to save: cloud or local"
+		},
+		"api-key": {
+			type: "string",
+			description: "gscdump user API key for --mode cloud; defaults to GSCDUMP_API_KEY"
+		},
+		"api-root": {
+			type: "string",
+			description: "Cloud API root for --mode cloud; defaults to GSCDUMP_API_ROOT or https://gscdump.com/api"
 		},
 		...OUTPUT_ARGS
 	},
 	async run({ args }) {
 		applyOutputMode(args);
 		const config = await loadConfig();
+		const mode = useCliRuntime().authModeOverride;
+		if (mode === "cloud") {
+			await loginCloud(args);
+			printNextSteps();
+			return;
+		}
 		if (config.clientId && config.clientSecret && !args.force) {
 			logger.info("Already configured");
 			const tokens = await loadTokens();
@@ -64,20 +90,23 @@ const initCommand = defineCommand({
 				const isExpired = tokens.expiry_date && tokens.expiry_date < Date.now();
 				if (isExpired && tokens.refresh_token) logger.info("Tokens expired but refresh available. Any live command will auto-refresh, or run `gscdump auth refresh`.");
 				else if (isExpired) logger.warn("Tokens expired without a refresh token. Run `gscdump auth login` to re-authenticate.");
-				else logger.info("Next: `gscdump sites` to list properties, or `gscdump query --help`.");
 			}
+			if (mode === "local") await saveAuthentication({ _tag: "Local" });
 			logger.info("Run `gscdump init --force` to reconfigure.");
+			printNextSteps();
 			return;
 		}
 		const byok = resolveBYOK();
 		if (byok) {
-			const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
+			const dataDir = args.store ? config.dataDir ?? defaultDataDir() : config.dataDir;
 			await saveConfig({
 				...config,
-				dataDir: dataDir ?? config.dataDir
+				...dataDir ? { dataDir } : {}
 			});
-			logger.success(`BYOK detected (${typeof byok === "string" ? "access-token" : "refresh-token"}) — auth setup skipped`);
-			logger.success("Setup complete! Run gscdump to get started.");
+			await saveAuthentication({ _tag: "Local" });
+			logger.success(`BYOK detected (${typeof byok === "string" ? "access-token" : "refresh-token"}). Auth setup skipped.`);
+			logger.success("Setup complete.");
+			printNextSteps();
 			return;
 		}
 		const envFile = await loadEnvFile();
@@ -108,27 +137,56 @@ const initCommand = defineCommand({
 				refresh_token: creds.refresh_token || envRef,
 				expiry_date: creds.expiry_date
 			});
+			await saveAuthentication({ _tag: "Local" });
 			console.log();
-			logger.success("Setup complete using .env credentials! Run gscdump to get started.");
+			logger.success("Setup complete using .env credentials.");
+			printNextSteps();
+			return;
+		}
+		if (!canPrompt()) {
+			if (!await loadTokens()) {
+				logger.error([
+					"No Google credentials found. Init cannot prompt without a terminal.",
+					"Run `gscdump auth login` in a terminal, or set GSC_CLIENT_ID, GSC_CLIENT_SECRET and GSC_REFRESH_TOKEN.",
+					"To use gscdump.com, run `gscdump init --mode cloud --api-key <key>`."
+				].join("\n"));
+				process.exit(1);
+			}
+			await saveConfig({
+				...config,
+				dataDir: config.dataDir ?? defaultDataDir()
+			});
+			await saveAuthentication({ _tag: "Local" });
+			logger.success("Setup complete with the saved Google login.");
+			printNextSteps();
 			return;
 		}
 		console.log();
 		console.log("  \x1B[1mWelcome to GSCDump!\x1B[0m");
 		console.log("  \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
 		console.log();
-		const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
+		const dataDir = args.store ? await promptDataDir(config.dataDir) : void 0;
 		await saveConfig({
 			...config,
 			...dataDir ? { dataDir } : {}
 		});
-		await smokeTest(await getAuth({ interactive: true }));
+		const sites = await runSmokeTest(await getAuth({ interactive: true }));
+		await saveAuthentication({ _tag: "Local" });
 		if (config.clientId && config.clientSecret) await maybeWriteEnvFile(config.clientId, config.clientSecret);
 		console.log();
-		logger.success("Setup complete! Run gscdump to get started.");
+		logger.success("Setup complete.");
+		printNextSteps(sites);
 	}
 });
-async function smokeTest(oauth) {
-	await runSmokeTest(oauth);
+function printNextSteps(sites = []) {
+	const site = sites.length === 1 ? sites[0] : "<site>";
+	console.log();
+	console.log("  Next:");
+	const line = (command, note) => console.log(`    ${command.padEnd(44)} # ${note}`);
+	if (sites.length !== 1) line("gscdump sites", "list your Sites");
+	line(`gscdump sync --site ${site}`, "fetch the last 28 days, then catch up on each run");
+	line(`gscdump sync --status --site ${site}`, "see what the Store holds");
+	console.log();
 }
 async function runSmokeTest(oauth) {
 	const sites = await googleSearchConsole(oauth).sites().catch((e) => e);
@@ -141,18 +199,19 @@ async function runSmokeTest(oauth) {
 			const url = project ? `https://console.developers.google.com/apis/api/searchconsole.googleapis.com/overview?project=${project}` : "https://console.developers.google.com/apis/api/searchconsole.googleapis.com/overview";
 			logger.info(`Enable it here, then retry: ${url}`);
 			logger.info("Note: it can take a few minutes to propagate after enabling.");
-			return;
+			return [];
 		}
 		if (/insufficient|scope|forbidden|403/i.test(msg)) {
 			logger.warn(`Smoke test failed (likely missing scopes): ${msg}`);
 			logger.info("Run `gscdump auth login --force` to re-consent with the required scopes.");
-			return;
+			return [];
 		}
 		logger.warn(`Smoke test failed: ${msg}`);
 		logger.info("Auth saved, but verify via `gscdump auth status` / `gscdump doctor`.");
-		return;
+		return [];
 	}
 	logger.success(`Verified: ${sites.length} GSC site(s) accessible`);
+	return sites.flatMap((site) => site.siteUrl ? [site.siteUrl] : []);
 }
 async function maybeWriteEnvFile(clientId, clientSecret) {
 	const tokens = await loadTokens();
@@ -182,4 +241,4 @@ async function maybeWriteEnvFile(clientId, clientSecret) {
 	await fs.writeFile(envPath, content, { mode: 384 });
 	logger.success(`Wrote ${displayPath(envPath)}`);
 }
-export { initCommand, runSmokeTest };
+export { initCommand, printNextSteps, runSmokeTest };