@gscdump/cli 0.37.2 → 0.37.4

package/dist/_chunks/doctor.mjs

@@ -0,0 +1,400 @@
+import { loadConfig } from "./config.mjs";
+import { resolveCliEnvironment } from "./environment.mjs";
+import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "./utils.mjs";
+import { parseEnvFile } from "./env-file.mjs";
+import { loadTokens, resolveAuth, resolveBYOK } from "./auth.mjs";
+import { createCommandContext, createLocalStore } from "./context.mjs";
+import process from "node:process";
+import { defineCommand } from "citty";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { googleSearchConsole, hasGscWriteScope, hasIndexingScope } from "gscdump/api";
+import { ofetch } from "ofetch";
+const REQUIRED_SCOPES = [
+	"https://www.googleapis.com/auth/webmasters",
+	"https://www.googleapis.com/auth/indexing",
+	"https://www.googleapis.com/auth/siteverification"
+];
+const FETCH_TIMEOUT_MS = 5e3;
+const TIME_SKEW_WARN_MS = 5 * 6e4;
+const WATERMARK_STALE_DAYS_WARN = 7;
+const RELEVANT_ENV_KEYS = [
+	"GSC_ACCESS_TOKEN",
+	"GSC_CLIENT_ID",
+	"GSC_CLIENT_SECRET",
+	"GSC_REFRESH_TOKEN",
+	"GOOGLE_ACCESS_TOKEN",
+	"GOOGLE_CLIENT_ID",
+	"GOOGLE_CLIENT_SECRET",
+	"GOOGLE_REFRESH_TOKEN",
+	"GOOGLE_APPLICATION_CREDENTIALS",
+	"GSC_SERVICE_ACCOUNT_JSON"
+];
+function redact(v) {
+	if (!v) return "<missing>";
+	if (v.length <= 6) return "***";
+	return `***${v.slice(-6)}`;
+}
+function hasGoogleScope(scopes, scope) {
+	const suffix = scope.replace("https://www.googleapis.com/auth/", "");
+	return scopes.includes(scope) || scopes.includes(suffix);
+}
+function missingRequiredScopes(scopes) {
+	return REQUIRED_SCOPES.filter((scope) => {
+		if (scope.endsWith("/webmasters")) return !hasGscWriteScope(scopes);
+		if (scope.endsWith("/indexing")) return !hasIndexingScope(scopes);
+		return !hasGoogleScope(scopes, scope);
+	});
+}
+async function checkEnv() {
+	const envPath = path.join(process.cwd(), ".env");
+	const parsed = parseEnvFile(envPath);
+	if (!parsed) return {
+		checks: [{
+			name: "env",
+			status: "info",
+			detail: "no .env (using shell env / saved tokens)"
+		}],
+		envKeys: /* @__PURE__ */ new Set()
+	};
+	const relevant = RELEVANT_ENV_KEYS.filter((k) => parsed[k] !== void 0);
+	const envKeys = new Set(relevant);
+	if (relevant.length === 0) return {
+		checks: [{
+			name: "env",
+			status: "info",
+			detail: `${displayPath(envPath)} found, no auth vars`
+		}],
+		envKeys
+	};
+	const inventory = relevant.map((k) => {
+		const v = parsed[k];
+		if (k.endsWith("CLIENT_ID")) return `${k}=${v}`;
+		return `${k}=${redact(v)}`;
+	});
+	return {
+		checks: [{
+			name: "env",
+			status: "info",
+			detail: `${displayPath(envPath)} → ${inventory.join(", ")} (not validated — see auth)`
+		}],
+		envKeys
+	};
+}
+function describeAuthSource(envKeys, byok) {
+	if (!byok) return "saved tokens";
+	const isAccessToken = typeof byok === "string";
+	const driver = isAccessToken ? "GSC_ACCESS_TOKEN" : "GSC_REFRESH_TOKEN";
+	const source = (isAccessToken ? ["GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN"] : ["GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN"]).some((k) => envKeys.has(k)) ? ".env" : "shell env";
+	return `BYOK ${isAccessToken ? "(access-token)" : "(refresh-token)"} from ${source} via ${driver}`;
+}
+async function checkAuth(envKeys) {
+	const checks = [];
+	const byok = resolveBYOK();
+	const tokens = await loadTokens();
+	if (!byok && !tokens) {
+		checks.push({
+			name: "auth",
+			status: "fail",
+			detail: "no BYOK env vars and no saved tokens; run `gscdump init`"
+		});
+		return {
+			checks,
+			liveToken: null
+		};
+	}
+	const clientId = resolveCliEnvironment().clientId ?? (await loadConfig()).clientId ?? null;
+	if (clientId) checks.push({
+		name: "auth.client_id",
+		status: "info",
+		detail: clientId
+	});
+	let liveToken = null;
+	let refreshError = null;
+	if (typeof byok === "string") liveToken = byok;
+	else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch((e) => {
+		refreshError = e?.data?.error_description ?? e?.data?.error ?? e?.message ?? String(e);
+		return null;
+	});
+	else if (tokens?.access_token) liveToken = tokens.access_token;
+	if (!liveToken) {
+		const source = describeAuthSource(envKeys, byok);
+		const detail = refreshError ? `${source} — refresh failed: ${refreshError} (token revoked / invalid — re-run \`gscdump auth login --force\` and update the source above)` : `${source} — no usable access token`;
+		checks.push({
+			name: "auth",
+			status: "fail",
+			detail
+		});
+		return {
+			checks,
+			liveToken: null
+		};
+	}
+	const info = await ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: liveToken } }).catch((e) => ({ error: e.message }));
+	if ("error" in info) {
+		checks.push({
+			name: "auth",
+			status: "fail",
+			detail: `tokeninfo failed: ${info.error}`
+		});
+		return {
+			checks,
+			liveToken: null
+		};
+	}
+	checks.push({
+		name: "auth",
+		status: "pass",
+		detail: describeAuthSource(envKeys, byok)
+	});
+	if (info.email) checks.push({
+		name: "auth.account",
+		status: "pass",
+		detail: info.email
+	});
+	const scopes = info.scope ? info.scope.split(/\s+/) : [];
+	const missing = missingRequiredScopes(scopes);
+	if (missing.length > 0) checks.push({
+		name: "auth.scopes",
+		status: "warn",
+		detail: `missing: ${missing.join(", ")} — \`gscdump auth login --force\` to re-consent`
+	});
+	else checks.push({
+		name: "auth.scopes",
+		status: "pass",
+		detail: `${scopes.length} granted`
+	});
+	if (tokens?.expiry_date) {
+		const expiresInMs = tokens.expiry_date - Date.now();
+		if (expiresInMs < 0) checks.push({
+			name: "auth.expiry",
+			status: "warn",
+			detail: `expired ${new Date(tokens.expiry_date).toISOString()} — will refresh on next call`
+		});
+		else checks.push({
+			name: "auth.expiry",
+			status: "pass",
+			detail: `valid for ${Math.floor(expiresInMs / 6e4)}m`
+		});
+	}
+	return {
+		checks,
+		liveToken
+	};
+}
+async function checkTimeSkew() {
+	const dateHeader = await ofetch.raw("https://oauth2.googleapis.com/tokeninfo", {
+		method: "GET",
+		timeout: FETCH_TIMEOUT_MS
+	}).then((r) => r.headers.get("date")).catch((e) => e?.response?.headers?.get("date") ?? null);
+	if (!dateHeader) return [{
+		name: "time",
+		status: "warn",
+		detail: "could not probe Google clock (no Date header)"
+	}];
+	const remoteMs = Date.parse(dateHeader);
+	if (!Number.isFinite(remoteMs)) return [{
+		name: "time",
+		status: "warn",
+		detail: `unparseable Date header: ${dateHeader}`
+	}];
+	const skewMs = Date.now() - remoteMs;
+	const human = `${skewMs >= 0 ? "+" : ""}${(skewMs / 1e3).toFixed(1)}s`;
+	if (Math.abs(skewMs) > TIME_SKEW_WARN_MS) return [{
+		name: "time",
+		status: "warn",
+		detail: `local clock ${human} off Google — OAuth refresh may reject; sync your clock`
+	}];
+	return [{
+		name: "time",
+		status: "pass",
+		detail: `in sync (${human})`
+	}];
+}
+async function checkDataDir(dataDir) {
+	const display = displayPath(dataDir);
+	const stat = await fs.stat(dataDir).catch(() => null);
+	if (!stat) return [{
+		name: "dataDir",
+		status: "warn",
+		detail: `${display} does not exist (will be created on first sync)`
+	}];
+	if (!stat.isDirectory()) return [{
+		name: "dataDir",
+		status: "fail",
+		detail: `${display} is not a directory`
+	}];
+	const probe = `${dataDir}/.gscdump-doctor-probe`;
+	return await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false) ? [{
+		name: "dataDir",
+		status: "pass",
+		detail: display
+	}] : [{
+		name: "dataDir",
+		status: "fail",
+		detail: `${display} not writable`
+	}];
+}
+async function checkStoreWatermarks(dataDir) {
+	if (!(await fs.stat(dataDir).catch(() => null))?.isDirectory()) return [{
+		name: "store.watermarks",
+		status: "pass",
+		detail: "no store yet (run `gscdump sync`)"
+	}];
+	const store = createLocalStore({ dataDir });
+	const watermarks = await store.engine.getWatermarks({ userId: store.userId }).catch(() => null);
+	if (!watermarks || watermarks.length === 0) return [{
+		name: "store.watermarks",
+		status: "pass",
+		detail: "no watermarks (run `gscdump sync`)"
+	}];
+	const bySite = /* @__PURE__ */ new Map();
+	for (const w of watermarks) {
+		const site = w.siteId ?? "(global)";
+		const existing = bySite.get(site);
+		if (!existing || w.newestDateSynced > existing) bySite.set(site, w.newestDateSynced);
+	}
+	const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+	const stale = [];
+	let freshest = null;
+	for (const [site, newest] of bySite) {
+		const days = Math.floor((Date.parse(today) - Date.parse(newest)) / 864e5);
+		if (days > WATERMARK_STALE_DAYS_WARN) stale.push({
+			site,
+			days
+		});
+		if (!freshest || days < freshest.days) freshest = {
+			site,
+			days
+		};
+	}
+	if (stale.length > 0) {
+		const sample = stale.slice(0, 3).map((s) => `${s.site} (${s.days}d)`).join(", ");
+		const more = stale.length > 3 ? ` +${stale.length - 3} more` : "";
+		return [{
+			name: "store.watermarks",
+			status: "warn",
+			detail: `${stale.length}/${bySite.size} site(s) stale >${WATERMARK_STALE_DAYS_WARN}d: ${sample}${more}`
+		}];
+	}
+	const tail = freshest ? `, freshest ${freshest.days}d ago` : "";
+	return [{
+		name: "store.watermarks",
+		status: "pass",
+		detail: `${bySite.size} site(s)${tail}`
+	}];
+}
+async function checkApiReachable(name, url) {
+	const reachable = await ofetch.raw(url, {
+		method: "GET",
+		timeout: FETCH_TIMEOUT_MS
+	}).then(() => true).catch(() => false);
+	return [{
+		name,
+		status: reachable ? "pass" : "warn",
+		detail: reachable ? "reachable" : `${new URL(url).hostname} unreachable (network/firewall?)`
+	}];
+}
+async function checkGscSites() {
+	const config = await loadConfig();
+	const auth = await resolveAuth({
+		interactive: false,
+		config
+	}).catch(() => null);
+	if (!auth) return [{
+		name: "gsc.sites",
+		status: "warn",
+		detail: "skipped (no usable auth)"
+	}];
+	const sites = await googleSearchConsole(auth).sites().catch((e) => e);
+	if (sites instanceof Error) return [{
+		name: "gsc.sites",
+		status: "fail",
+		detail: `sites() failed: ${sites.message}`
+	}];
+	const checks = [];
+	const verified = sites.filter((s) => s.permissionLevel !== "siteUnverifiedUser").length;
+	checks.push({
+		name: "gsc.sites",
+		status: "pass",
+		detail: `${sites.length} site(s) accessible (${verified} verified)`
+	});
+	if (config.defaultSite) {
+		const match = sites.find((s) => s.siteUrl === config.defaultSite || (s.siteUrl ?? "").includes(String(config.defaultSite)));
+		checks.push(match ? {
+			name: "config.defaultSite",
+			status: "pass",
+			detail: `${config.defaultSite} ✓`
+		} : {
+			name: "config.defaultSite",
+			status: "fail",
+			detail: `${config.defaultSite} not in verified site list`
+		});
+	}
+	return checks;
+}
+const doctorCommand = defineCommand({
+	meta: {
+		name: "doctor",
+		description: "Run health checks (env, auth, scopes, time, dataDir, store, GSC reachability + ping, defaultSite)"
+	},
+	args: { ...OUTPUT_ARGS },
+	async run({ args }) {
+		const { json } = applyOutputMode(args);
+		const { dataDir } = await createCommandContext();
+		const envResult = await checkEnv();
+		const [authResult, timeChecks, dataDirChecks, watermarkChecks, gscApi, indexingApi, siteVerificationApi] = await Promise.all([
+			checkAuth(envResult.envKeys),
+			checkTimeSkew(),
+			checkDataDir(dataDir),
+			checkStoreWatermarks(dataDir),
+			checkApiReachable("gsc.api", "https://searchconsole.googleapis.com/$discovery/rest?version=v1"),
+			checkApiReachable("indexing.api", "https://indexing.googleapis.com/$discovery/rest?version=v3"),
+			checkApiReachable("siteverification.api", "https://www.googleapis.com/discovery/v1/apis/siteVerification/v1/rest")
+		]);
+		const sitesChecks = authResult.liveToken ? await checkGscSites() : [{
+			name: "gsc.sites",
+			status: "warn",
+			detail: "skipped (auth failed)"
+		}];
+		const all = [
+			...envResult.checks,
+			...authResult.checks,
+			...timeChecks,
+			...dataDirChecks,
+			...watermarkChecks,
+			...gscApi,
+			...indexingApi,
+			...siteVerificationApi,
+			...sitesChecks
+		];
+		if (json) {
+			console.log(JSON.stringify({
+				checks: all,
+				ok: all.every((c) => c.status !== "fail")
+			}, null, 2));
+			return;
+		}
+		const ICONS = {
+			pass: "\x1B[32m✓\x1B[0m",
+			warn: "\x1B[33m!\x1B[0m",
+			fail: "\x1B[31m✗\x1B[0m",
+			info: "\x1B[34mℹ\x1B[0m"
+		};
+		console.log();
+		for (const c of all) {
+			const detail = c.detail ? ` \x1B[90m${c.detail}\x1B[0m` : "";
+			console.log(`  ${ICONS[c.status]} ${c.name}${detail}`);
+		}
+		console.log();
+		const failed = all.filter((c) => c.status === "fail");
+		if (failed.length > 0) {
+			logger.error(`${failed.length} check(s) failed`);
+			process.exit(1);
+		}
+		const warned = all.filter((c) => c.status === "warn");
+		if (warned.length > 0) logger.warn(`${warned.length} warning(s)`);
+		else logger.success("All checks passed");
+	}
+});
+export { doctorCommand };

package/dist/_chunks/dump.mjs

@@ -0,0 +1,193 @@
+import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseSearchType, toCSV } from "./utils.mjs";
+import { allTables, createCommandContext } from "./context.mjs";
+import { readParquetRows } from "./native-duckdb.mjs";
+import process from "node:process";
+import { defineCommand } from "citty";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { Buffer } from "node:buffer";
+const DEFAULT_OUT = "./gscdump-export";
+const FORMATS = [
+	"parquet",
+	"json",
+	"ndjson",
+	"csv"
+];
+const dumpCommand = defineCommand({
+	meta: {
+		name: "dump",
+		description: "Export live Parquet files from the local store to a directory"
+	},
+	args: {
+		"site": {
+			type: "string",
+			alias: "s",
+			description: "Site URL (e.g., sc-domain:example.com); ignored with --all-sites"
+		},
+		"out": {
+			type: "string",
+			alias: "o",
+			default: DEFAULT_OUT,
+			description: `Output directory (default: ${DEFAULT_OUT})`
+		},
+		"format": {
+			type: "string",
+			alias: "F",
+			default: "parquet",
+			description: `Output format: ${FORMATS.join(", ")} (default: parquet copies raw files)`
+		},
+		"tables": {
+			type: "string",
+			alias: "t",
+			description: `Comma-separated table list (default: all). Known: ${allTables().join(", ")}`
+		},
+		"all-sites": {
+			type: "boolean",
+			default: false,
+			description: "Iterate every site with local data"
+		},
+		"compact": {
+			type: "boolean",
+			default: false,
+			description: "Compact every closed month into a single file before exporting"
+		},
+		"search-type": {
+			type: "string",
+			description: `Restrict dump to a single GSC search-type slice (${ALL_SEARCH_TYPES.join(", ")}). Default: all slices.`
+		},
+		...OUTPUT_ARGS
+	},
+	async run({ args }) {
+		const { json, quiet } = applyOutputMode(args);
+		const format = String(args.format);
+		if (!FORMATS.includes(format)) {
+			logger.error(`Invalid --format: ${format}. Allowed: ${FORMATS.join(", ")}`);
+			process.exit(1);
+		}
+		const tablesFilter = args.tables ? new Set(String(args.tables).split(",").map((t) => t.trim()).filter(Boolean)) : null;
+		const searchType = parseSearchType(args["search-type"]);
+		const ctx = await createCommandContext({
+			needsAuth: !args["all-sites"],
+			needsStore: true
+		});
+		const store = ctx.store;
+		const outDir = path.resolve(String(args.out));
+		const targets = args["all-sites"] ? await listSitesWithData(store) : [await ctx.resolveSite(args.site ? String(args.site) : void 0)];
+		if (targets.length === 0) {
+			logger.warn("No sites with local data. Run `gscdump sync` first.");
+			process.exit(0);
+		}
+		if (args.compact) for (const siteUrl of targets) await compactClosedMonths(store, siteUrl, quiet);
+		const summary = [];
+		for (const siteUrl of targets) {
+			const entries = (await listLiveEntries(store, siteUrl, searchType)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
+			if (entries.length === 0) {
+				if (!quiet) logger.warn(`No data for ${siteUrl}; skipping`);
+				continue;
+			}
+			if (format === "parquet") {
+				const written = await dumpParquet(store, entries, outDir);
+				summary.push({
+					site: siteUrl,
+					files: written,
+					rows: 0,
+					format,
+					outPath: outDir
+				});
+			} else {
+				const written = await dumpRowFormat(store, entries, outDir, siteUrl, format);
+				summary.push({
+					site: siteUrl,
+					files: written.files,
+					rows: written.rows,
+					format,
+					outPath: outDir
+				});
+			}
+		}
+		if (json) {
+			console.log(JSON.stringify({
+				outDir,
+				sites: summary
+			}, null, 2));
+			return;
+		}
+		for (const s of summary) {
+			const rows = s.rows ? `, ${s.rows.toLocaleString()} rows` : "";
+			logger.success(`[${s.site}] ${s.files} ${s.format} file(s)${rows} → ${displayPath(s.outPath)}`);
+		}
+	}
+});
+async function listSitesWithData(store) {
+	const siteIds = /* @__PURE__ */ new Set();
+	for (const table of allTables()) {
+		const entries = await store.engine.listLive({
+			userId: store.userId,
+			table
+		});
+		for (const e of entries) if (e.siteId) siteIds.add(e.siteId);
+	}
+	return Array.from(siteIds);
+}
+async function listLiveEntries(store, siteUrl, searchType) {
+	const siteId = store.siteIdFor(siteUrl);
+	return (await Promise.all(allTables().map((table) => store.engine.listLive({
+		userId: store.userId,
+		siteId,
+		table,
+		...searchType !== void 0 ? { searchType } : {}
+	})))).flat();
+}
+async function dumpParquet(store, entries, outDir) {
+	await fs.mkdir(outDir, { recursive: true });
+	let copied = 0;
+	for (const entry of entries) {
+		const bytes = await store.engine.readObject(entry.objectKey);
+		const target = path.join(outDir, entry.objectKey);
+		await fs.mkdir(path.dirname(target), { recursive: true });
+		await fs.writeFile(target, Buffer.from(bytes));
+		copied++;
+	}
+	return copied;
+}
+async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
+	const byTable = /* @__PURE__ */ new Map();
+	for (const e of entries) {
+		const arr = byTable.get(e.table) ?? [];
+		arr.push(e);
+		byTable.set(e.table, arr);
+	}
+	const safeSite = siteUrl.replace(/[^a-z0-9]+/gi, "_");
+	const siteDir = path.join(outDir, safeSite);
+	await fs.mkdir(siteDir, { recursive: true });
+	let files = 0;
+	let totalRows = 0;
+	for (const [table, tableEntries] of byTable) {
+		const rows = await readParquetRows(tableEntries.map((e) => path.join(store.dataDir, e.objectKey)), table);
+		const ext = format === "csv" ? "csv" : format === "ndjson" ? "ndjson" : "json";
+		const target = path.join(siteDir, `${table}.${ext}`);
+		let body;
+		if (format === "json") body = JSON.stringify(rows, null, 2);
+		else if (format === "ndjson") body = rows.map((r) => JSON.stringify(r)).join("\n");
+		else body = rows.length > 0 ? toCSV(rows, Object.keys(rows[0])) : "";
+		await fs.writeFile(target, body);
+		files++;
+		totalRows += rows.length;
+	}
+	return {
+		files,
+		rows: totalRows
+	};
+}
+async function compactClosedMonths(store, siteUrl, quiet) {
+	const siteId = store.siteIdFor(siteUrl);
+	for (const table of allTables()) {
+		if (!quiet) logger.info(`Compacting ${table} (raw→d7→d30→d90)`);
+		await store.engine.compactTiered({
+			userId: store.userId,
+			siteId,
+			table
+		});
+	}
+}
+export { dumpCommand };