@gscdump/cli 3.7.0 → 4.0.0

package/dist/config.mjs

@@ -2,8 +2,8 @@ import { useCliRuntime } from "./runtime.mjs";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { z } from "zod";
-import os from "node:os";
 import { SearchTypes } from "gscdump/query";
+import os from "node:os";
 function setConfigDir(dir) {
 	useCliRuntime().configDir = dir;
 }
@@ -14,9 +14,7 @@ const configSchema = z.strictObject({
 	clientId: z.string().optional(),
 	clientSecret: z.string().optional(),
 	defaultSite: z.string().optional(),
-	defaultPeriod: z.string().optional(),
 	defaultFormat: z.enum(["json", "csv"]).optional(),
-	defaultDb: z.string().optional(),
 	dataDir: z.string().optional(),
 	defaultLimit: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(),
 	defaultSearchType: z.enum(SearchTypes).optional(),
@@ -27,7 +25,12 @@ const configSchema = z.strictObject({
 	]).optional(),
 	serviceAccountPath: z.string().optional()
 });
-const RETIRED_KEYS = ["mode", "cloudUrl"];
+const RETIRED_KEYS = [
+	"mode",
+	"cloudUrl",
+	"defaultPeriod",
+	"defaultDb"
+];
 function dropRetiredKeys(value) {
 	if (typeof value !== "object" || value === null || Array.isArray(value)) return value;
 	return Object.fromEntries(Object.entries(value).filter(([key]) => !RETIRED_KEYS.includes(key)));

package/dist/context.mjs

@@ -1,16 +1,20 @@
+import { openQuotaLedger, quotaFetchOptions } from "./quota-ledger.mjs";
 import { resolveAuthentication } from "./auth-state.mjs";
 import { loadResolvedConfig } from "./config.mjs";
-import { logger } from "./utils.mjs";
 import { resolveAuth } from "./auth.mjs";
 import { createCloudGoogleClient } from "./cloud-google.mjs";
 import { createLocalStore } from "./local-store.mjs";
+import { listStoreSites } from "./store-sites.mjs";
 import process from "node:process";
 import { cancel, isCancel, select } from "@clack/prompts";
 import { googleSearchConsole } from "gscdump/client";
+import { parseGscSiteUrl, resolveSiteInput } from "gscdump";
+const FULL_SITE_URL_RE = /^(?:sc-domain:|https?:\/\/)/i;
 async function createCommandContext(opts = {}) {
-	const { needsAuth = false, needsStore = false, interactive = false, byok, fetchOptions } = opts;
+	const { needsAuth = false, needsStore = false, interactive = false, byok, quota = true } = opts;
 	const { config, dataDir } = await loadResolvedConfig();
 	const authentication = needsAuth ? await resolveAuthentication() : { _tag: "Local" };
+	const fetchOptions = needsAuth && quota ? quotaFetchOptions(await openQuotaLedger({ dataDir }), opts.fetchOptions) : opts.fetchOptions;
 	const auth = needsAuth && authentication._tag === "Local" ? await resolveAuth({
 		interactive,
 		config,
@@ -25,27 +29,35 @@ async function createCommandContext(opts = {}) {
 			permissionLevel: s.permissionLevel || "unknown"
 		}));
 	};
-	const resolveSite = async (target) => {
-		const hint = target ?? config.defaultSite;
-		const sites = await loadSites();
-		if (sites.length === 0) {
-			logger.error("No verified sites found");
-			process.exit(1);
-		}
+	const storeCandidates = async () => (await listStoreSites(dataDir)).map((site) => ({
+		siteUrl: site.siteUrl,
+		inStore: true
+	}));
+	const asksGoogle = (options) => options.scope !== "store" && client !== null;
+	const matchSite = async (input, options = {}) => {
+		const local = await storeCandidates();
+		const fromStore = resolveSiteInput(input, local);
+		if (!asksGoogle(options) || fromStore.kind === "resolved" && (fromStore.via === "exact" || !FULL_SITE_URL_RE.test(input.trim()))) return fromStore;
+		return resolveSiteInput(input, [...local, ...await loadSites()]);
+	};
+	const resolveSite = async (target, options = {}) => {
+		const hint = target?.trim() || config.defaultSite;
+		const askGoogle = asksGoogle(options);
 		if (hint) {
-			const exact = sites.find((site) => site.siteUrl === hint);
-			if (exact) return exact.siteUrl;
-			const matches = sites.filter((site) => site.siteUrl.includes(hint));
-			if (matches.length === 1) return matches[0].siteUrl;
-			if (matches.length > 1) throw new Error(`Multiple Sites match "${hint}". Use an exact Site URL from gscdump sites.`);
-			throw new Error(`No verified Site matches "${hint}". Run gscdump sites to list available Sites.`);
+			const resolution = await matchSite(hint, options);
+			if (resolution.kind === "resolved") return resolution.siteUrl;
+			throw new Error(formatSiteResolution(resolution, askGoogle ? "account" : "store"));
 		}
-		if (sites.length === 1) return sites[0].siteUrl;
+		const local = await storeCandidates();
+		const pool = [...new Set([...local, ...askGoogle ? await loadSites() : []].map((site) => site.siteUrl))];
+		if (pool.length === 0) throw new Error(askGoogle ? "No verified Sites found. Add the Site in Search Console, then run `gscdump sites`." : "The Store has no data. Run `gscdump sync --site example.com` first.");
+		if (pool.length === 1) return pool[0];
+		if (!process.stdin.isTTY || !process.stdout.isTTY) throw new Error(`Pass --site. Sites: ${pool.join(", ")}`);
 		const selected = await select({
-			message: "Select a site",
-			options: sites.map((s) => ({
-				value: s.siteUrl,
-				label: s.siteUrl
+			message: "Select a Site",
+			options: pool.map((siteUrl) => ({
+				value: siteUrl,
+				label: siteUrl
 			}))
 		});
 		if (isCancel(selected)) {
@@ -62,7 +74,25 @@ async function createCommandContext(opts = {}) {
 		client,
 		store,
 		loadSites,
+		matchSite,
 		resolveSite
 	};
 }
-export { createCommandContext };
+function siteArg(siteUrl) {
+	return siteUrl.startsWith("http://") ? siteUrl : parseGscSiteUrl(siteUrl).displayLabel;
+}
+function formatSiteResolution(resolution, scope) {
+	switch (resolution.kind) {
+		case "ambiguous": return `"${resolution.input}" matches more than one Site: ${resolution.candidates.join(", ")}. Pass one of these Site URLs to --site.`;
+		case "covered-by-parent":
+			if (scope === "store") return `"${resolution.input}" is part of the Site ${resolution.parent}. The Store keeps page paths only. Pass --live to filter by host.`;
+			return `"${resolution.input}" is part of the Site ${resolution.parent}. Filter its pages instead: --site ${siteArg(resolution.parent)} --page ~${resolution.input.trim().replace(/^[a-z]+:\/\//i, "")}.`;
+		case "not-found":
+			if (scope === "store") {
+				const sync = `gscdump sync --site ${siteArg(resolution.input.trim())}`;
+				return resolution.known.length === 0 ? `The Store has no data. Run \`${sync}\` first.` : `The Store has no data for "${resolution.input}". Local Sites: ${resolution.known.join(", ")}. Run \`${sync}\` to add it.`;
+			}
+			return resolution.known.length === 0 ? `No Site matches "${resolution.input}". Your account has no verified Sites.` : `No Site matches "${resolution.input}". Sites: ${resolution.known.join(", ")}.`;
+	}
+}
+export { createCommandContext, formatSiteResolution, siteArg };

package/dist/coverage.mjs

@@ -0,0 +1,251 @@
+import { quotaLedgerPath, quotaStatus, readLedgerState } from "./quota-ledger.mjs";
+import { INSPECTION_QPD_PER_PROPERTY } from "./inspection-record.mjs";
+import { inspectionCandidates, loadInspectionState } from "./local-entities.mjs";
+import { datesForJob, jobWindowStart } from "./sync-plan.mjs";
+import { getDateRange, getLatestGscDate, getOldestGscDate, getPstDate } from "gscdump/dates";
+import { createSitemapListStore, createSitemapReadStore, parseSitemapFeedIdentity } from "@gscdump/engine/entities";
+function partialOrComplete(done, total, failed, extra = {}) {
+	if (total === 0) return { kind: "empty" };
+	if (done >= total && failed === 0) return {
+		kind: "complete",
+		done
+	};
+	return {
+		kind: "partial",
+		done,
+		total,
+		failed,
+		pending: Math.max(0, total - done - failed),
+		...extra
+	};
+}
+function analyticsCoverage(input) {
+	const byJob = /* @__PURE__ */ new Map();
+	for (const state of input.states) {
+		const searchType = state.searchType ?? "web";
+		const key = `${state.table}|${searchType}`;
+		const job = byJob.get(key) ?? {
+			table: state.table,
+			searchType,
+			states: []
+		};
+		job.states.push({
+			date: state.date,
+			state: state.state
+		});
+		byJob.set(key, job);
+	}
+	const jobs = [];
+	const dayState = /* @__PURE__ */ new Map();
+	const window = {
+		kind: "catch-up",
+		firstStart: input.latest,
+		floor: input.floor,
+		end: input.latest
+	};
+	for (const job of byJob.values()) {
+		const from = jobWindowStart(window, job.states);
+		const dates = datesForJob(job.table, getDateRange(from, input.latest), input.today);
+		const stateByDate = new Map(job.states.map((state) => [state.date, state.state]));
+		let done = 0;
+		let failed = 0;
+		for (const date of dates) {
+			const state = stateByDate.get(date);
+			const prior = dayState.get(date);
+			if (state === "done") {
+				done++;
+				if (!prior) dayState.set(date, "done");
+			} else if (state === "failed") {
+				failed++;
+				dayState.set(date, "failed");
+			} else if (prior !== "failed") dayState.set(date, "pending");
+		}
+		jobs.push({
+			table: job.table,
+			searchType: job.searchType,
+			from: dates[0] ?? from,
+			to: input.latest,
+			coverage: partialOrComplete(done, dates.length, failed)
+		});
+	}
+	jobs.sort((a, b) => a.table.localeCompare(b.table) || a.searchType.localeCompare(b.searchType));
+	const sortedDays = [...dayState.keys()].sort();
+	const values = [...dayState.values()];
+	return {
+		jobs,
+		days: {
+			from: sortedDays[0] ?? input.latest,
+			to: input.latest,
+			coverage: partialOrComplete(values.filter((v) => v === "done").length, values.length, values.filter((v) => v === "failed").length)
+		}
+	};
+}
+function inspectionCoverage(input) {
+	const { inspected } = input;
+	const total = new Set(input.candidates).size;
+	const done = [...new Set(input.candidates)].filter((url) => inspected.has(url)).length;
+	const pending = total - done;
+	const perRun = Math.max(0, Math.min(input.perRun, INSPECTION_QPD_PER_PROPERTY));
+	const etaRuns = perRun > 0 ? Math.ceil(pending / perRun) : void 0;
+	return {
+		perRun,
+		coverage: partialOrComplete(done, total, 0, {
+			...etaRuns !== void 0 ? {
+				etaRuns,
+				etaDays: etaRuns
+			} : {},
+			...input.blockedUntil !== void 0 ? { resumesAt: input.blockedUntil } : {}
+		})
+	};
+}
+function datasetKey(table, searchType) {
+	return `${table}|${searchType ?? "web"}`;
+}
+function syncStateCoverageReader(states) {
+	const done = /* @__PURE__ */ new Map();
+	for (const state of states) {
+		if (state.state !== "done") continue;
+		const key = datasetKey(state.table, state.searchType);
+		done.set(key, [...done.get(key) ?? [], state.date]);
+	}
+	return { datesForTable: async ({ table, searchType, start, end }) => (done.get(datasetKey(table, searchType)) ?? []).filter((date) => date >= start && date <= end) };
+}
+function hasSyncedDays(states, table, searchType) {
+	const key = datasetKey(table, searchType);
+	return states.some((state) => state.state === "done" && datasetKey(state.table, state.searchType) === key);
+}
+function n(value) {
+	return value.toLocaleString("en-US");
+}
+function plural(count, one, many) {
+	return `${n(count)} ${count === 1 ? one : many}`;
+}
+function clock(at) {
+	return new Date(at).toLocaleString("en-US", {
+		hour: "numeric",
+		minute: "2-digit",
+		month: "short",
+		day: "numeric"
+	});
+}
+function nextCommand(coverage) {
+	if (coverage.run.kind === "running") return `gscdump sync --status --site ${coverage.site}`;
+	return `gscdump sync --site ${coverage.site}`;
+}
+function analyticsLine(coverage) {
+	const { from, to, coverage: c } = coverage.days;
+	if (c.kind === "empty") return `Analytics: nothing synced yet. A first sync fetches the last 28 days.`;
+	if (c.kind === "complete") return `Analytics: all ${plural(c.done, "day", "days")} from ${from} to ${to}, every table.`;
+	const failed = c.failed > 0 ? ` ${plural(c.failed, "day has", "days have")} a failed table.` : "";
+	const next = coverage.analyticsBlockedUntil !== void 0 ? ` Google paused Search Analytics calls until ${clock(coverage.analyticsBlockedUntil)}. The next sync after that continues.` : " The next sync continues from there.";
+	return `Analytics: ${n(c.done)} of ${plural(c.total, "day", "days")} so far (${from} to ${to}).${failed}${next}`;
+}
+function inspectionLine(coverage) {
+	const { coverage: c, perRun } = coverage.inspections;
+	if (c.kind === "empty") return "Inspections: no URLs yet. Sync saves sitemaps and pages first.";
+	if (c.kind === "complete") return `Inspections: all ${plural(c.done, "URL", "URLs")} inspected at least once.`;
+	const head = `Inspections: ${n(c.done)} of ${plural(c.total, "URL", "URLs")} so far.`;
+	const blocked = c.resumesAt !== void 0 ? ` Google allows more after ${clock(c.resumesAt)}.` : "";
+	if (perRun === 0) return `${head}${blocked} Pass --inspect-limit to inspect URLs during sync.`;
+	const eta = c.etaRuns ?? 0;
+	if (perRun >= 2e3) return `${head}${blocked} Daily sync covers the rest in about ${plural(eta, "day", "days")} at ${n(perRun)} URLs a day.`;
+	const fastest = Math.ceil(c.pending / INSPECTION_QPD_PER_PROPERTY);
+	const raise = fastest < eta ? ` Pass --inspect-limit ${n(INSPECTION_QPD_PER_PROPERTY).replace(",", "")} to finish in about ${plural(fastest, "day", "days")}.` : "";
+	return `${head}${blocked} Daily sync covers the rest in about ${plural(eta, "run", "runs")} at ${n(perRun)} URLs a run.${raise}`;
+}
+function sitemapLine(coverage) {
+	const { coverage: c, urls } = coverage.sitemaps;
+	if (c.kind === "empty") return "Sitemaps: none saved yet.";
+	if (c.kind === "complete") return `Sitemaps: all ${plural(c.done, "sitemap", "sitemaps")} saved, ${plural(urls, "URL", "URLs")}.`;
+	return `Sitemaps: ${n(c.done)} of ${plural(c.total, "sitemap", "sitemaps")} saved so far, ${plural(urls, "URL", "URLs")}. The next sync reads the rest.`;
+}
+function renderCoverage(coverage) {
+	const lines = [
+		analyticsLine(coverage),
+		inspectionLine(coverage),
+		sitemapLine(coverage)
+	];
+	if (coverage.run.kind === "running") {
+		const { record } = coverage.run;
+		lines.unshift(`Sync running: ${n(record.done)}/${n(record.planned)} days, pid ${record.pid}.`);
+	} else if (coverage.run.kind === "stale") {
+		const { record } = coverage.run;
+		lines.unshift(`The last sync stopped at ${n(record.done)}/${n(record.planned)} days without finishing (pid ${record.pid}). The next sync retries its dates.`);
+	}
+	lines.push(`Next: ${nextCommand(coverage)}`);
+	return lines;
+}
+async function sitemapCoverage(store, siteUrl) {
+	const ctx = {
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl)
+	};
+	const list = await createSitemapListStore({ dataSource: store.dataSource }).load(ctx);
+	if (!list) return {
+		coverage: { kind: "empty" },
+		urls: 0
+	};
+	const feeds = /* @__PURE__ */ new Set();
+	let urls = 0;
+	const iterated = await createSitemapReadStore({ dataSource: store.dataSource }).iterateSitemapGenerationUrls(ctx);
+	if (iterated._tag === "iterator") for await (const record of iterated.items) {
+		feeds.add(record.feedpath);
+		urls++;
+	}
+	const expected = list.sitemaps.filter((sitemap) => sitemap.contents.some((content) => content.submitted > 0));
+	const saved = expected.filter((sitemap) => {
+		const identity = parseSitemapFeedIdentity(sitemap.path);
+		return identity._tag === "ok" && feeds.has(identity.url);
+	}).length;
+	if (list.sitemaps.length === 0) return {
+		coverage: {
+			kind: "complete",
+			done: 0
+		},
+		urls
+	};
+	return {
+		coverage: partialOrComplete(saved, expected.length, 0),
+		urls
+	};
+}
+async function readStoreCoverage(input) {
+	const { store, site } = input;
+	const now = input.now ?? /* @__PURE__ */ new Date();
+	const ctx = {
+		userId: store.userId,
+		siteId: store.siteIdFor(site)
+	};
+	const ledger = input.ledger ?? await readLedgerState(quotaLedgerPath(store.dataDir));
+	const { jobs: analytics, days } = analyticsCoverage({
+		states: await store.engine.getSyncStates(ctx),
+		latest: getLatestGscDate(),
+		floor: getOldestGscDate(),
+		today: getPstDate(now)
+	});
+	const inspectBlock = quotaStatus(ledger, {
+		api: "urlInspection",
+		site,
+		now
+	}).blocked;
+	const analyticsBlock = quotaStatus(ledger, {
+		api: "searchAnalytics",
+		site,
+		now
+	}).blocked;
+	return {
+		site,
+		analytics,
+		days,
+		inspections: inspectionCoverage({
+			candidates: await inspectionCandidates(store, site),
+			inspected: new Set((await loadInspectionState(store.dataSource, ctx, now)).latest.keys()),
+			perRun: input.inspectLimit,
+			...inspectBlock ? { blockedUntil: inspectBlock.until } : {}
+		}),
+		sitemaps: await sitemapCoverage(store, site),
+		...analyticsBlock ? { analyticsBlockedUntil: analyticsBlock.until } : {},
+		run: input.run
+	};
+}
+export { analyticsCoverage, hasSyncedDays, inspectionCoverage, nextCommand, readStoreCoverage, renderCoverage, syncStateCoverageReader };

package/dist/dump-bing.mjs

@@ -0,0 +1,71 @@
+import { resolveAuthentication } from "./auth-state.mjs";
+import { urlInProperty } from "./local-entities.mjs";
+import { getBingClient, inspectBingCredentials } from "./bing-auth.mjs";
+import { dumpBingSite, parseBingDumpOptions, unwrapBing } from "./bing-data.mjs";
+import { dumpHostedBingSite, listHostedBingSites } from "./bing-hosted.mjs";
+import fs from "node:fs/promises";
+import path from "node:path";
+function matchBingSites(bingSites, googleSites) {
+	if (googleSites === "all") return [...bingSites];
+	return bingSites.filter((bing) => googleSites.some((google) => urlInProperty(google, bing) || urlInProperty(bing, google)));
+}
+async function withSizes(summary) {
+	const files = await Promise.all(summary.files.map(async (file) => ({
+		dataset: file.dataset,
+		path: file.path,
+		bytes: (await fs.stat(file.path)).size,
+		rows: file.rows
+	})));
+	return {
+		siteUrl: summary.siteUrl,
+		files
+	};
+}
+function message(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+async function dumpBing(opts) {
+	const options = parseBingDumpOptions({ format: opts.format === "csv" || opts.format === "ndjson" ? opts.format : "json" });
+	const bingDir = path.join(opts.outDir, "bing");
+	const sites = [];
+	const failures = [];
+	const authentication = await resolveAuthentication();
+	if (authentication._tag === "Cloud") {
+		const hosted = (await listHostedBingSites(authentication)).filter((site) => site.connection._tag === "connected");
+		const wanted = new Set(matchBingSites(hosted.map((site) => site.siteUrl), opts.googleSites));
+		const selected = hosted.filter((site) => wanted.has(site.siteUrl));
+		if (selected.length === 0) return {
+			_tag: "skipped",
+			reason: "No connected Bing site matches the dumped sites."
+		};
+		for (const site of selected) await dumpHostedBingSite(authentication, site, bingDir, options).then(async (summary) => sites.push(await withSizes(summary))).catch((error) => failures.push({
+			siteUrl: site.siteUrl,
+			error: message(error)
+		}));
+		return {
+			_tag: "dumped",
+			sites,
+			failures
+		};
+	}
+	if ((await inspectBingCredentials())._tag === "Missing") return {
+		_tag: "skipped",
+		reason: "No Bing login. Run `gscdump bing login` to include Bing."
+	};
+	const client = await getBingClient();
+	const selected = matchBingSites(unwrapBing(await client.getUserSites({ signal: AbortSignal.timeout(3e4) })).filter((site) => site.isVerified).map((site) => site.url), opts.googleSites);
+	if (selected.length === 0) return {
+		_tag: "skipped",
+		reason: "No verified Bing site matches the dumped sites."
+	};
+	for (const siteUrl of selected) await dumpBingSite(client, siteUrl, bingDir, options).then(async (summary) => sites.push(await withSizes(summary))).catch((error) => failures.push({
+		siteUrl,
+		error: message(error)
+	}));
+	return {
+		_tag: "dumped",
+		sites,
+		failures
+	};
+}
+export { dumpBing, matchBingSites };