@gscdump/cli 3.8.0 → 4.0.0

package/dist/commands/sync.mjs

@@ -1,24 +1,45 @@
 import { syncCommandMeta } from "../command-meta.mjs";
-import { OUTPUT_ARGS, applyOutputMode, clearLine, displayPath, formatAge, logger, parseIntegerOption, progressBar, runWithConcurrency } from "../utils.mjs";
+import { QUOTA_CAPS, googleErrorMessage, openQuotaLedger, parseQuotaRefusal } from "../quota-ledger.mjs";
+import { OUTPUT_ARGS, applyOutputMode, clearLine, displayPath, formatAge, logger, parseIntegerOption, parseNameList, progressBar, runWithConcurrency } from "../utils.mjs";
 import { TABLE_DIMS, allTables, assembleDatesRow, createLocalStore } from "../local-store.mjs";
-import { createCommandContext } from "../context.mjs";
+import { formatSiteIdCollision, readSiteMap, recordStoreSite, siteUrlForId } from "../store-sites.mjs";
+import { createCommandContext, formatSiteResolution } from "../context.mjs";
+import { INSPECTION_QPD_PER_PROPERTY } from "../inspection-record.mjs";
+import { inspectionCandidates, syncInspections, syncSitemaps } from "../local-entities.mjs";
+import { minimumCallsPerDate, planJobDates, planSyncJobs, resolveWindow } from "../sync-plan.mjs";
+import { analyticsCoverage, readStoreCoverage, renderCoverage } from "../coverage.mjs";
+import { isProcessAlive, readSyncRun, startSyncRun, syncRunStatus } from "../sync-run.mjs";
+import { loadSitemapUrls } from "../sitemap.mjs";
+import { createRequestPacer } from "../request-pacer.mjs";
 import process from "node:process";
 import { defineCommand } from "citty";
+import { randomUUID } from "node:crypto";
 import { SearchTypes } from "gscdump/query";
+import { getLatestGscDate, getOldestGscDate, getPstDate, groupIntoRanges } from "gscdump/dates";
+import { runGscSearchAppearanceContextSlice, runGscSyncSlice } from "@gscdump/engine-gsc-api";
+import { resolveSiteInput } from "gscdump";
 import { createRowAccumulator } from "@gscdump/engine/ingest";
 import { createEmptyTypesStore } from "@gscdump/engine/entities";
-import { daysAgoUtc, getDateRange } from "gscdump/dates";
 import { DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/engine/rollups";
-const DEFAULT_TABLES = [
-	"pages",
-	"queries",
-	"countries",
-	"dates"
-];
-const DEFAULT_TYPES = ["web"];
 const ALL_SEARCH_TYPES = Object.values(SearchTypes);
-const DEFAULT_PENDING_DAYS = 3;
+const DEFAULT_TABLES = allTables();
+const DEFAULT_TYPES = ALL_SEARCH_TYPES;
+const GSC_ROW_LIMIT = 25e3;
+const MAX_SLICE_ROWS = 5e6;
+const SLICE_TABLES = [
+	"search_appearance",
+	"search_appearance_pages",
+	"search_appearance_queries",
+	"search_appearance_page_queries",
+	"hourly_pages"
+];
+function isSliceTable(table) {
+	return SLICE_TABLES.includes(table);
+}
 const DEFAULT_CONCURRENCY = 8;
+const DEFAULT_MAX_IN_FLIGHT = 8;
+const DEFAULT_REQUESTS_PER_MINUTE = 600;
+const INSPECT_CONCURRENCY = 4;
 const EMPTY_TYPE_PROBE_MIN_DAYS = 7;
 const EMPTY_TYPE_PROTECTED = ["web"];
 function createProgressTracker(total, quiet) {
@@ -48,25 +69,33 @@ function createProgressTracker(total, quiet) {
 		}
 	};
 }
-async function syncTable(store, siteUrl, table, searchType, dates, client, concurrency, force, progress) {
+const SYNC_STOP = Symbol("syncStop");
+function stopError(stop) {
+	return Object.assign(new Error(describeStop(stop)), { [SYNC_STOP]: stop });
+}
+function stopOf(error) {
+	return error?.[SYNC_STOP];
+}
+function describeStop(stop) {
+	if (stop.kind === "budget") return `Reached --max-calls ${stop.maxCalls}.`;
+	return `Google quota: ${stop.reason}`;
+}
+async function syncTable(opts) {
+	const { store, siteUrl, job, control, progress } = opts;
+	const { table, type: searchType } = job;
 	const dims = TABLE_DIMS[table];
 	const siteId = store.siteIdFor(siteUrl);
-	let totalRows = 0;
-	let skipped = 0;
-	let failed = 0;
-	const priorStates = await store.engine.getSyncStates({
-		userId: store.userId,
-		siteId,
-		table,
-		searchType
-	});
-	const stateByDate = new Map(priorStates.map((s) => [s.date, s]));
-	const label = searchType === "web" ? table : `${table}/${searchType}`;
-	await runWithConcurrency(dates, concurrency, async (date) => {
-		const prior = stateByDate.get(date);
-		if (!force && prior?.state === "done") {
-			skipped++;
-			progress.tick(`${label} ${date} (skip)`);
+	const totals = {
+		rows: 0,
+		skipped: opts.skipped,
+		failed: 0,
+		pending: 0,
+		fetched: 0
+	};
+	await runWithConcurrency(opts.dates, opts.concurrency, async (date) => {
+		if (control.stop) {
+			totals.pending++;
+			progress.tick(`${job.label} ${date} (later)`);
 			return;
 		}
 		const scope = {
@@ -77,28 +106,42 @@ async function syncTable(store, siteUrl, table, searchType, dates, client, concu
 			searchType
 		};
 		await store.engine.setSyncState(scope, "inflight");
-		const result = await runOneDate(store, client, siteUrl, table, searchType, dims, date).catch((err) => ({
+		const result = await runOneDate(store, opts.client, siteUrl, table, searchType, dims, date, opts.latest).catch((err) => ({
 			kind: "error",
 			error: err
 		}));
 		if (result.kind === "error") {
-			await store.engine.setSyncState(scope, "failed", { error: result.error.message });
-			failed++;
-			progress.tick(`${label} ${date} (fail)`);
+			const stop = stopOf(result.error);
+			if (stop) {
+				control.stop ??= stop;
+				await store.engine.setSyncState(scope, "pending", { error: describeStop(stop) });
+				totals.pending++;
+				progress.tick(`${job.label} ${date} (later)`);
+				return;
+			}
+			await store.engine.setSyncState(scope, "failed", { error: googleErrorMessage(result.error) });
+			totals.failed++;
+			progress.tick(`${job.label} ${date} (fail)`);
+			opts.run.tick();
 			return;
 		}
-		await store.engine.setSyncState(scope, "done");
-		totalRows += result.rows;
-		progress.tick(`${label} ${date}`);
+		await store.engine.setSyncState(scope, result.final ? "done" : "pending");
+		totals.rows += result.rows;
+		totals.fetched++;
+		if (!result.final) totals.pending++;
+		progress.tick(`${job.label} ${date}`);
+		opts.run.tick();
 	});
-	return {
-		rows: totalRows,
-		skipped,
-		failed
-	};
+	return totals;
+}
+function isFinalDate(date, latest, metadata) {
+	if (date > latest) return false;
+	const incompleteDate = metadata?.first_incomplete_date;
+	if (incompleteDate && incompleteDate <= date) return false;
+	const incompleteHour = metadata?.first_incomplete_hour;
+	return !(incompleteHour && incompleteHour.slice(0, 10) <= date);
 }
 async function fetchDateRows(client, siteUrl, searchType, dims, date) {
-	const rowLimit = 25e3;
 	const rows = [];
 	let startRow = 0;
 	while (true) {
@@ -107,7 +150,7 @@ async function fetchDateRows(client, siteUrl, searchType, dims, date) {
 			endDate: date,
 			dimensions: dims,
 			searchType,
-			rowLimit,
+			rowLimit: GSC_ROW_LIMIT,
 			startRow
 		})).rows || [];
 		for (const apiRow of batch) rows.push({
@@ -117,12 +160,13 @@ async function fetchDateRows(client, siteUrl, searchType, dims, date) {
 			ctr: apiRow.ctr ?? 0,
 			position: apiRow.position ?? 0
 		});
-		if (batch.length === 0) break;
+		if (batch.length < GSC_ROW_LIMIT) break;
 		startRow += batch.length;
 	}
 	return rows;
 }
-async function runOneDate(store, client, siteUrl, table, searchType, dims, date) {
+async function runOneDate(store, client, siteUrl, table, searchType, dims, date, latest) {
+	if (isSliceTable(table)) return runSliceDate(store, client, siteUrl, table, searchType, date, latest);
 	const apiRows = await fetchDateRows(client, siteUrl, searchType, dims, date);
 	let rows = [];
 	if (table === "dates") {
@@ -137,6 +181,63 @@ async function runOneDate(store, client, siteUrl, table, searchType, dims, date)
 		accumulator.push(table, apiRows);
 		rows = accumulator.drain().get(table)?.get(date) ?? [];
 	}
+	return writeDayRows(store, siteUrl, table, searchType, date, rows, isFinalDate(date, latest, void 0));
+}
+async function runSliceDate(store, client, siteUrl, table, searchType, date, latest) {
+	const base = {
+		client,
+		siteUrl,
+		startDate: date,
+		endDate: date,
+		searchType,
+		rowLimit: GSC_ROW_LIMIT,
+		cpuBudgetMs: Infinity
+	};
+	const drainDay = (accumulator) => {
+		if (accumulator.overflowed) throw new Error(`${table} ${date}: more rows than one sync can hold`);
+		return accumulator.drain().get(table)?.get(date) ?? [];
+	};
+	let rows;
+	let metadata;
+	if (table === "search_appearance" || table === "hourly_pages") {
+		const accumulator = createRowAccumulator({
+			maxRows: MAX_SLICE_ROWS,
+			date
+		});
+		const result = await runGscSyncSlice({
+			...base,
+			table,
+			onBatch: async (batch) => {
+				accumulator.push(table, batch);
+			}
+		});
+		if (result.hasMore) throw new Error(`${table} ${date}: Google stopped before the last page`);
+		metadata = result.metadata;
+		rows = drainDay(accumulator);
+	} else {
+		const byAppearance = /* @__PURE__ */ new Map();
+		const result = await runGscSearchAppearanceContextSlice({
+			...base,
+			table,
+			onContextBatch: async ({ searchAppearance, rows: batch }) => {
+				let accumulator = byAppearance.get(searchAppearance);
+				if (!accumulator) {
+					accumulator = createRowAccumulator({
+						maxRows: MAX_SLICE_ROWS,
+						searchAppearance
+					});
+					byAppearance.set(searchAppearance, accumulator);
+				}
+				accumulator.push(table, batch);
+			}
+		});
+		if (result.hasMore) throw new Error(`${table} ${date}: Google stopped before the last page`);
+		metadata = result.metadata;
+		rows = [...byAppearance.values()].flatMap(drainDay);
+	}
+	return writeDayRows(store, siteUrl, table, searchType, date, rows, isFinalDate(date, latest, metadata));
+}
+async function writeDayRows(store, siteUrl, table, searchType, date, rows, final) {
 	const writeCtx = {
 		userId: store.userId,
 		siteId: store.siteIdFor(siteUrl),
@@ -147,16 +248,25 @@ async function runOneDate(store, client, siteUrl, table, searchType, dims, date)
 	await store.engine.writeDay(writeCtx, rows);
 	return {
 		kind: "ok",
-		rows: rows.length
+		rows: rows.length,
+		final
 	};
 }
+const LEDGER_FETCH_OPTIONS = { onResponseError(ctx) {
+	if (ctx.options.retryStatusCodes) ctx.options.retryStatusCodes = ctx.options.retryStatusCodes.filter((status) => status !== 403);
+} };
 const syncCommand = defineCommand({
 	meta: syncCommandMeta,
 	args: {
 		"site": {
 			type: "string",
 			alias: "s",
-			description: "Site URL"
+			description: "Site, for example example.com"
+		},
+		"all-sites": {
+			type: "boolean",
+			default: false,
+			description: "Sync every verified Site, one after another (they share one Google quota)"
 		},
 		"start": {
 			type: "string",
@@ -164,11 +274,11 @@ const syncCommand = defineCommand({
 		},
 		"end": {
 			type: "string",
-			description: "End date (YYYY-MM-DD); defaults to 3 days ago"
+			description: "End date (YYYY-MM-DD); defaults to the latest date Google has finalized"
 		},
 		"days": {
 			type: "string",
-			description: `Number of days back to sync (default: ${DEFAULT_PENDING_DAYS})`
+			description: "Sync the last N final days instead of catching up"
 		},
 		"tables": {
 			type: "string",
@@ -184,14 +294,35 @@ const syncCommand = defineCommand({
 			default: false,
 			description: "Ignore stored empty-type markers and re-probe every requested type"
 		},
-		"no-rollups": {
+		"rollups": {
 			type: "boolean",
-			default: false,
-			description: "Skip the post-sync rollup rebuild (daily/weekly totals, top-N tables)"
+			default: true,
+			description: "Rebuild rollups after sync (daily and weekly totals, top-N tables)",
+			negativeDescription: "Skip the post-sync rollup rebuild"
+		},
+		"sitemaps": {
+			type: "boolean",
+			default: true,
+			description: "Save the Search Console sitemap list and the URLs in each sitemap",
+			negativeDescription: "Skip saving sitemaps"
+		},
+		"inspections": {
+			type: "boolean",
+			default: true,
+			description: "Run URL Inspection on URLs that are due and save the results",
+			negativeDescription: "Skip URL Inspection"
+		},
+		"inspect-limit": {
+			type: "string",
+			description: `Most URLs to inspect in this run (default: 50; Google allows ${INSPECTION_QPD_PER_PROPERTY} per Site per day)`
 		},
 		"full": {
 			type: "boolean",
-			description: "Backfill up to 450 days of available Search Console data"
+			description: `Backfill all the Search Console data Google keeps (16 months, plus 14 days Google often still serves)`
+		},
+		"max-calls": {
+			type: "string",
+			description: "Most Search Analytics calls this run makes. The next run continues where it stopped."
 		},
 		...OUTPUT_ARGS,
 		"force": {
@@ -202,12 +333,16 @@ const syncCommand = defineCommand({
 		"status": {
 			type: "boolean",
 			default: false,
-			description: "Print watermarks + sync-state summary instead of syncing"
+			description: "Print coverage, gaps, and the running sync instead of syncing"
 		},
 		"concurrency": {
 			type: "string",
 			alias: "c",
-			description: `Concurrent in-flight day fetches per table (default: ${DEFAULT_CONCURRENCY})`
+			description: `Concurrent day fetches per table (default: ${DEFAULT_CONCURRENCY}). All tables share ${DEFAULT_MAX_IN_FLIGHT} requests in flight.`
+		},
+		"requests-per-minute": {
+			type: "string",
+			description: `Most Search Analytics requests to start per minute, across all tables (default: ${DEFAULT_REQUESTS_PER_MINUTE}; Google allows 1,200)`
 		},
 		"serial-tables": {
 			type: "boolean",
@@ -217,7 +352,7 @@ const syncCommand = defineCommand({
 		"retry-failed": {
 			type: "boolean",
 			default: false,
-			description: "Only re-run dates currently in `failed` state (cheaper than --force)"
+			description: "Only re-run dates currently in `failed` state (a plain sync also retries them)"
 		},
 		"dry-run": {
 			type: "boolean",
@@ -229,283 +364,618 @@ const syncCommand = defineCommand({
 		const { json, quiet } = applyOutputMode(args);
 		const days = parseIntegerOption(args.days, "--days");
 		const concurrency = parseIntegerOption(args.concurrency, "--concurrency") ?? DEFAULT_CONCURRENCY;
+		const maxCalls = parseIntegerOption(args["max-calls"], "--max-calls");
+		const inspectLimit = Math.min(parseIntegerOption(args["inspect-limit"], "--inspect-limit", 0) ?? 50, INSPECTION_QPD_PER_PROPERTY);
+		const tables = args.tables ? parseNameList(args.tables, allTables(), "--tables") : DEFAULT_TABLES;
+		const requestedTypes = args.types ? parseNameList(args.types, ALL_SEARCH_TYPES, "--types") : DEFAULT_TYPES;
 		if (args.status) {
 			const ctx = await createCommandContext();
+			const siteUrl = args.site ? await resolveStatusSite(ctx, String(args.site)) : void 0;
 			await printSyncStatus({
 				config: ctx.config,
 				dataDir: ctx.dataDir
-			}, args.site ? String(args.site) : void 0, json);
+			}, siteUrl, json, inspectLimit);
 			return;
 		}
 		const ctx = await createCommandContext({
 			needsAuth: true,
-			needsStore: true
+			needsStore: true,
+			fetchOptions: LEDGER_FETCH_OPTIONS,
+			quota: false
 		});
-		const client = ctx.client;
-		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
-		const tables = args.tables ? String(args.tables).split(",").map((t) => t.trim()).filter(isKnownTable) : DEFAULT_TABLES;
-		const requestedTypes = args.types ? String(args.types).split(",").map((t) => t.trim()).filter(isKnownSearchType) : DEFAULT_TYPES;
-		if (requestedTypes.length === 0) {
-			logger.error(`No valid search types specified. Allowed: ${ALL_SEARCH_TYPES.join(",")}`);
-			process.exit(1);
-		}
 		const store = ctx.store;
-		const siteId = store.siteIdFor(siteUrl);
-		const scope = {
-			userId: store.userId,
-			siteId
-		};
-		const emptyTypesStore = createEmptyTypesStore({ dataSource: store.dataSource });
-		let emptyTypesDoc = await emptyTypesStore.load(scope);
-		if (emptyTypesDoc.emptyTypes.length > 0) {
-			const entries = await store.engine.listLive(scope);
-			const populated = new Set(entries.filter((entry) => entry.rowCount > 0).map((entry) => entry.searchType ?? "web"));
-			const toClear = requestedTypes.filter((type) => emptyTypesDoc.emptyTypes.includes(type) && populated.has(type));
-			if (toClear.length > 0) {
-				if (!args["dry-run"]) emptyTypesDoc = await emptyTypesStore.clear(scope, toClear);
-				else emptyTypesDoc.emptyTypes = emptyTypesDoc.emptyTypes.filter((type) => !toClear.includes(type));
-			}
-		}
-		const forceTypes = Boolean(args["force-types"]);
-		const skippedTypes = [];
-		const types = [];
-		for (const t of requestedTypes) {
-			if (!forceTypes && emptyTypesDoc.emptyTypes.includes(t) && !EMPTY_TYPE_PROTECTED.includes(t)) {
-				skippedTypes.push(t);
+		const requestedSites = args["all-sites"] ? (await ctx.loadSites()).map((site) => site.siteUrl) : [await ctx.resolveSite(args.site ? String(args.site) : void 0)];
+		const siteUrls = [];
+		for (const siteUrl of requestedSites) {
+			const claim = await recordStoreSite(store.dataDir, siteUrl, {
+				userId: store.userId,
+				write: !args["dry-run"]
+			});
+			if (claim.ok) {
+				siteUrls.push(siteUrl);
 				continue;
 			}
-			types.push(t);
+			if (!args["all-sites"]) {
+				logger.error(formatSiteIdCollision(claim.error));
+				process.exit(1);
+			}
+			logger.warn(`Skipped ${siteUrl}. ${formatSiteIdCollision(claim.error)}`);
 		}
-		if (skippedTypes.length > 0 && !quiet) logger.info(`Skipping ${skippedTypes.join(", ")} (marked empty for this site; pass --force-types to re-probe).`);
-		const endDate = args.end ? String(args.end) : daysAgoUtc(DEFAULT_PENDING_DAYS);
-		let startDate;
-		if (args.start) startDate = String(args.start);
-		else if (args.full) startDate = daysAgoUtc(450);
-		else if (days !== void 0) startDate = daysAgoUtc(days + DEFAULT_PENDING_DAYS - 1);
-		else startDate = daysAgoUtc(5);
-		let dates = getDateRange(startDate, endDate);
-		if (dates.length === 0) {
-			logger.error(`No dates to sync (start=${startDate}, end=${endDate})`);
-			process.exit(1);
+		const latest = getLatestGscDate();
+		const window = resolveWindow({
+			start: args.start ? String(args.start) : void 0,
+			end: args.end ? String(args.end) : void 0,
+			days,
+			full: Boolean(args.full),
+			latest,
+			floor: getOldestGscDate()
+		});
+		const mode = args["retry-failed"] ? "retry-failed" : args.force ? "force" : "resume";
+		if (!args["dry-run"]) {
+			const existing = syncRunStatus(await readSyncRun(store.dataDir), {
+				now: Date.now(),
+				isAlive: isProcessAlive
+			});
+			if (existing.kind === "running") {
+				logger.error(`Another sync is running (pid ${existing.record.pid}, ${existing.record.done}/${existing.record.planned} days). Wait for it to finish, or run \`gscdump sync --status\`.`);
+				process.exit(1);
+			}
 		}
-		const printCompletion = async (status, totals, reason, rollupError) => {
-			if (!json) return;
-			console.log(JSON.stringify({
-				status,
-				...reason ? { reason } : {},
-				siteUrl,
-				range: {
-					start: startDate,
-					end: endDate
-				},
-				tables,
-				types,
-				skippedTypes,
-				totals,
-				watermarks: await store.engine.getWatermarks({
-					userId: store.userId,
-					siteId
-				}),
-				...rollupError ? { rollupError } : {}
-			}, null, 2));
+		const ledger = await openQuotaLedger({ dataDir: store.dataDir });
+		const pacer = createRequestPacer({
+			maxInFlight: DEFAULT_MAX_IN_FLIGHT,
+			perMinute: Math.min(parseIntegerOption(args["requests-per-minute"], "--requests-per-minute") ?? DEFAULT_REQUESTS_PER_MINUTE, QUOTA_CAPS.searchAnalytics.perMinute)
+		});
+		const common = {
+			store,
+			client: gatedClient(ctx.client, {
+				pacer,
+				ledger,
+				calls: { made: 0 },
+				maxCalls
+			}),
+			ledger,
+			tables,
+			requestedTypes,
+			window,
+			mode,
+			latest,
+			concurrency,
+			inspectLimit,
+			json,
+			quiet,
+			args
 		};
-		if (types.length === 0) {
-			if (!quiet) logger.warn(`All requested types are marked empty. Pass --force-types to check them again.`);
-			await printCompletion("skipped", {}, "empty-types");
+		if (args["dry-run"]) {
+			const plans = [];
+			for (const siteUrl of siteUrls) plans.push(await planSite({
+				...common,
+				siteUrl
+			}));
+			if (json) console.log(JSON.stringify(plans.length === 1 ? plans[0] : { sites: plans }, null, 2));
 			return;
 		}
-		if (args["retry-failed"]) {
-			const failedSet = /* @__PURE__ */ new Set();
-			const selectedTables = new Set(tables);
-			const selectedTypes = new Set(types);
-			const states = await store.engine.getSyncStates({
-				userId: store.userId,
-				siteId
+		const run = await startSyncRun({
+			dataDir: store.dataDir,
+			sites: siteUrls,
+			onHeartbeat: () => ledger.flush()
+		});
+		const onInterrupt = () => {
+			clearLine();
+			logger.warn("Sync interrupted. Run the same command to resume.");
+			Promise.all([run.finish("interrupted"), ledger.flush()]).catch((error) => logger.warn(`Could not save sync progress: ${error.message}`)).finally(() => process.exit(130));
+		};
+		process.once("SIGINT", onInterrupt);
+		const results = [];
+		for (const siteUrl of siteUrls) {
+			run.setSite(siteUrl);
+			const result = await syncSite({
+				...common,
+				siteUrl,
+				run
 			});
-			for (const s of states) if (selectedTables.has(s.table) && selectedTypes.has(s.searchType ?? "web") && s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
-			dates = dates.filter((d) => failedSet.has(d));
-			if (dates.length === 0) {
-				if (!quiet) logger.success("No failed dates in range. Nothing to retry.");
-				await printCompletion("skipped", {}, "no-failed-dates");
-				return;
-			}
-			args.force = true;
-			if (!quiet) logger.info(`--retry-failed: ${dates.length} date(s) to retry`);
+			results.push(result);
+			if (result.stop?.kind === "budget") break;
 		}
-		if (args["dry-run"]) {
-			const plan = [];
-			for (const table of tables) for (const type of types) for (const date of dates) plan.push({
-				table,
-				searchType: type,
-				date
-			});
-			if (json) {
-				console.log(JSON.stringify({
-					siteUrl,
-					range: {
-						start: startDate,
-						end: endDate
-					},
-					tables,
-					types,
-					totalCalls: plan.length,
-					plan
-				}, null, 2));
-				return;
-			}
-			console.log();
-			logger.info(`Plan: ${plan.length} API call(s) for ${siteUrl}`);
-			console.log(`  Tables:   ${tables.join(", ")}`);
-			console.log(`  Types:    ${types.join(", ")}`);
-			console.log(`  Range:    ${startDate} → ${endDate} (${dates.length} days)`);
-			console.log();
-			logger.info("Pass without --dry-run to execute.");
-			return;
+		process.off("SIGINT", onInterrupt);
+		const failed = results.some((result) => result.status === "failed");
+		const stopped = results.some((result) => result.status === "partial");
+		await ledger.flush();
+		await run.finish(failed ? "failed" : stopped ? "stopped" : "completed");
+		if (json) {
+			const skippedSites = siteUrls.slice(results.length);
+			const reports = results.map((result) => result.report);
+			console.log(JSON.stringify(siteUrls.length === 1 ? reports[0] : {
+				sites: reports,
+				notStarted: skippedSites
+			}, null, 2));
+		} else if (siteUrls.length > results.length && !quiet) logger.info(`Not started: ${siteUrls.slice(results.length).join(", ")}. The next sync covers them.`);
+		if (failed) process.exit(1);
+	}
+});
+async function resolveTypes(opts, dryRun) {
+	const { store, siteUrl, requestedTypes, quiet } = opts;
+	const scope = {
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl)
+	};
+	const emptyTypesStore = createEmptyTypesStore({ dataSource: store.dataSource });
+	let emptyTypesDoc = await emptyTypesStore.load(scope);
+	if (emptyTypesDoc.emptyTypes.length > 0) {
+		const entries = await store.engine.listLive(scope);
+		const populated = new Set(entries.filter((entry) => entry.rowCount > 0).map((entry) => entry.searchType ?? "web"));
+		const toClear = requestedTypes.filter((type) => emptyTypesDoc.emptyTypes.includes(type) && populated.has(type));
+		if (toClear.length > 0) {
+			if (!dryRun) emptyTypesDoc = await emptyTypesStore.clear(scope, toClear);
+			else emptyTypesDoc.emptyTypes = emptyTypesDoc.emptyTypes.filter((type) => !toClear.includes(type));
 		}
-		if (!quiet) {
-			logger.info(`Syncing ${siteUrl} (${tables.join(", ")}) [${types.join(", ")}] → ${displayPath(store.dataDir)}`);
-			logger.info(`Range: ${startDate} → ${endDate} (${dates.length} days)`);
+	}
+	const forceTypes = Boolean(opts.args["force-types"]);
+	const skippedTypes = [];
+	const types = [];
+	for (const t of requestedTypes) {
+		if (!forceTypes && emptyTypesDoc.emptyTypes.includes(t) && !EMPTY_TYPE_PROTECTED.includes(t)) {
+			skippedTypes.push(t);
+			continue;
 		}
-		const serialTables = Boolean(args["serial-tables"]);
-		const start = Date.now();
-		const totals = {};
-		const jobs = [];
-		for (const table of tables) for (const type of types) {
-			const label = type === "web" ? table : `${table}/${type}`;
-			jobs.push({
-				table,
-				type,
-				label
+		types.push(t);
+	}
+	if (skippedTypes.length > 0 && !quiet) logger.info(`Skipping ${skippedTypes.join(", ")} (marked empty for this site; pass --force-types to re-probe).`);
+	return {
+		types,
+		skippedTypes,
+		emptyTypes: emptyTypesDoc.emptyTypes
+	};
+}
+async function buildSitePlan(opts, types, skippedTypes, emptyTypes) {
+	const { store, siteUrl, tables, quiet, args } = opts;
+	const { jobs, unsupported } = planSyncJobs(tables, types);
+	if (unsupported.length > 0 && !quiet && (args.tables || args.types)) logger.info(`Skipping ${unsupported.map((job) => job.label).join(", ")}: Google has no such breakdown for that search type.`);
+	const states = await store.engine.getSyncStates({
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl)
+	});
+	const today = getPstDate();
+	const forceTypes = Boolean(args["force-types"]);
+	return {
+		types,
+		skippedTypes,
+		jobs: jobs.map((job) => {
+			const jobStates = states.filter((state) => state.table === job.table && (state.searchType ?? "web") === job.type);
+			const mode = forceTypes && opts.mode === "resume" && emptyTypes.includes(job.type) ? "force" : opts.mode;
+			const planned = planJobDates({
+				table: job.table,
+				window: opts.window,
+				states: jobStates,
+				today,
+				mode
 			});
+			return {
+				job,
+				dates: planned.dates,
+				skipped: planned.skippedDone
+			};
+		})
+	};
+}
+function windowLabel(window) {
+	return window.kind === "range" ? `${window.start} → ${window.end}` : `catch up to ${window.end} (new tables start ${window.firstStart})`;
+}
+async function planSite(opts) {
+	const { types, skippedTypes, emptyTypes } = await resolveTypes(opts, true);
+	const items = (await buildSitePlan(opts, types, skippedTypes, emptyTypes)).jobs.flatMap(({ job, dates }) => dates.map((date) => ({
+		table: job.table,
+		searchType: job.type,
+		date,
+		calls: minimumCallsPerDate(job.table)
+	})));
+	const minimumCalls = items.reduce((sum, item) => sum + item.calls, 0);
+	const ranges = groupIntoRanges([...new Set(items.map((item) => item.date))]);
+	if (!opts.json) {
+		console.log();
+		logger.info(`Plan: at least ${minimumCalls.toLocaleString("en-US")} Search Analytics call(s) for ${opts.siteUrl}`);
+		console.log(`  Tables:   ${opts.tables.join(", ")}`);
+		console.log(`  Types:    ${types.join(", ")}`);
+		console.log(`  Window:   ${windowLabel(opts.window)}`);
+		console.log(`  Dates:    ${ranges.length === 0 ? "none; every date is done" : ranges.map((range) => range.startDate === range.endDate ? range.startDate : `${range.startDate}..${range.endDate}`).join(", ")}`);
+		console.log();
+		logger.info("Pass without --dry-run to execute. Google may need more calls for large days and search appearances.");
+	}
+	return {
+		siteUrl: opts.siteUrl,
+		window: opts.window,
+		tables: opts.tables,
+		types,
+		skippedTypes,
+		ranges,
+		minimumCalls,
+		totalCalls: minimumCalls,
+		plan: items
+	};
+}
+async function syncSite(opts) {
+	const { store, siteUrl, client, quiet, run } = opts;
+	const siteId = store.siteIdFor(siteUrl);
+	const scope = {
+		userId: store.userId,
+		siteId
+	};
+	const { types, skippedTypes, emptyTypes } = await resolveTypes(opts, false);
+	const forceTypes = Boolean(opts.args["force-types"]);
+	const report = async (status, totals, extra = {}) => ({
+		status,
+		siteUrl,
+		window: opts.window,
+		tables: opts.tables,
+		types,
+		skippedTypes,
+		totals,
+		watermarks: await store.engine.getWatermarks(scope),
+		...extra
+	});
+	if (types.length === 0) {
+		if (!quiet) logger.warn(`All requested types are marked empty. Pass --force-types to check them again.`);
+		return {
+			status: "skipped",
+			report: await report("skipped", {}, { reason: "empty-types" })
+		};
+	}
+	const plan = await buildSitePlan(opts, types, skippedTypes, emptyTypes);
+	const plannedDates = plan.jobs.reduce((sum, entry) => sum + entry.dates.length, 0);
+	if (opts.mode === "retry-failed" && plannedDates === 0) {
+		if (!quiet) logger.success("No failed dates in range. Nothing to retry.");
+		return {
+			status: "skipped",
+			report: await report("skipped", {}, { reason: "no-failed-dates" })
+		};
+	}
+	await store.engine.setSyncStates(plan.jobs.flatMap(({ job, dates }) => dates.map((date) => ({
+		userId: store.userId,
+		siteId,
+		table: job.table,
+		date,
+		searchType: job.type
+	}))), "pending");
+	run.addPlanned(plannedDates);
+	if (!quiet) {
+		const ranges = groupIntoRanges([...new Set(plan.jobs.flatMap((entry) => entry.dates))]);
+		logger.info(`Syncing ${siteUrl} (${opts.tables.join(", ")}) [${types.join(", ")}] → ${displayPath(store.dataDir)}`);
+		logger.info(ranges.length === 0 ? `Every date is done up to ${opts.latest}.` : `${plannedDates.toLocaleString("en-US")} table days, newest first: ${ranges.slice(-3).reverse().map((range) => range.startDate === range.endDate ? range.startDate : `${range.startDate}..${range.endDate}`).join(", ")}${ranges.length > 3 ? `, and ${ranges.length - 3} more range(s)` : ""}`);
+	}
+	const control = {};
+	const start = Date.now();
+	const totals = {};
+	const progress = createProgressTracker(plannedDates, quiet);
+	const runJob = (entry) => syncTable({
+		store,
+		siteUrl,
+		job: entry.job,
+		dates: entry.dates,
+		skipped: entry.skipped,
+		client,
+		concurrency: opts.concurrency,
+		latest: opts.latest,
+		control,
+		progress,
+		run
+	});
+	if (opts.args["serial-tables"]) for (const entry of plan.jobs) totals[entry.job.label] = await runJob(entry);
+	else {
+		const results = await Promise.all(plan.jobs.map(runJob));
+		plan.jobs.forEach((entry, i) => {
+			totals[entry.job.label] = results[i];
+		});
+	}
+	progress.done();
+	await run.save();
+	const seconds = ((Date.now() - start) / 1e3).toFixed(1);
+	if (!quiet) {
+		logger.success(`Synced ${siteUrl} in ${seconds}s`);
+		for (const [t, n] of Object.entries(totals)) {
+			const suffix = [
+				n.skipped > 0 ? `${n.skipped} already done` : null,
+				n.pending > 0 ? `${n.pending} for the next sync` : null,
+				n.failed > 0 ? `\x1B[31m${n.failed} failed\x1B[0m` : null
+			].filter(Boolean).join(", ");
+			const tail = suffix ? ` (${suffix})` : "";
+			console.log(`  ${t}: ${n.rows.toLocaleString()} rows${tail}`);
 		}
-		const progress = createProgressTracker(dates.length * jobs.length, quiet);
-		if (serialTables) for (const job of jobs) totals[job.label] = await syncTable(store, siteUrl, job.table, job.type, dates, client, concurrency, args.force || forceTypes, progress);
-		else {
-			const results = await Promise.all(jobs.map((job) => syncTable(store, siteUrl, job.table, job.type, dates, client, concurrency, args.force || forceTypes, progress)));
-			jobs.forEach((job, i) => {
-				totals[job.label] = results[i];
-			});
+		console.log();
+	}
+	if (control.stop?.kind === "budget" && !quiet) logger.info(`${describeStop(control.stop)} The next sync continues from here.`);
+	else if (control.stop?.kind === "quota" && !opts.json) logger.warn(`${describeStop(control.stop)} The next sync continues after ${clockTime(control.stop.resetsAt)}.`);
+	const anyFailed = Object.values(totals).some((t) => t.failed > 0);
+	if (!forceTypes && opts.tables.length > 0 && !control.stop) {
+		const toMark = [];
+		for (const type of types) {
+			if (EMPTY_TYPE_PROTECTED.includes(type)) continue;
+			const typeTotals = plan.jobs.filter((entry) => entry.job.type === type).map((entry) => totals[entry.job.label]);
+			if (!(typeTotals.length > 0 && typeTotals.every((t) => t.fetched >= EMPTY_TYPE_PROBE_MIN_DAYS && t.skipped === 0 && t.failed === 0 && t.rows === 0))) continue;
+			if ((await store.engine.listLive({
+				...scope,
+				searchType: type
+			})).some((entry) => entry.rowCount > 0)) continue;
+			toMark.push(type);
 		}
-		progress.done();
-		const seconds = ((Date.now() - start) / 1e3).toFixed(1);
-		if (!quiet) {
-			logger.success(`Synced ${siteUrl} in ${seconds}s`);
-			for (const [t, n] of Object.entries(totals)) {
-				const suffix = [n.skipped > 0 ? `${n.skipped} skipped` : null, n.failed > 0 ? `\x1B[31m${n.failed} failed\x1B[0m` : null].filter(Boolean).join(", ");
-				const tail = suffix ? ` (${suffix})` : "";
-				console.log(`  ${t}: ${n.rows.toLocaleString()} rows${tail}`);
-			}
-			console.log();
+		if (toMark.length > 0) {
+			await createEmptyTypesStore({ dataSource: store.dataSource }).mark(scope, toMark);
+			if (!quiet) logger.info(`Marked empty for future syncs: ${toMark.join(", ")} (0 rows across ${EMPTY_TYPE_PROBE_MIN_DAYS}+ days; pass --force-types to re-probe).`);
 		}
-		const anyFailed = Object.values(totals).some((t) => t.failed > 0);
-		const rowsByType = /* @__PURE__ */ new Map();
-		const failedByType = /* @__PURE__ */ new Map();
-		const skippedByType = /* @__PURE__ */ new Map();
-		for (const job of jobs) {
-			const t = totals[job.label];
-			rowsByType.set(job.type, (rowsByType.get(job.type) ?? 0) + t.rows);
-			failedByType.set(job.type, (failedByType.get(job.type) ?? 0) + t.failed);
-			skippedByType.set(job.type, (skippedByType.get(job.type) ?? 0) + t.skipped);
+	}
+	if (forceTypes && emptyTypes.length > 0) {
+		const toClear = types.filter((type) => emptyTypes.includes(type) && plan.jobs.some((entry) => entry.job.type === type && totals[entry.job.label].rows > 0));
+		if (toClear.length > 0) {
+			await createEmptyTypesStore({ dataSource: store.dataSource }).clear(scope, toClear);
+			if (!quiet) logger.info(`Cleared empty markers for: ${toClear.join(", ")} (re-probe found data).`);
 		}
-		if (!forceTypes && tables.length > 0 && dates.length >= EMPTY_TYPE_PROBE_MIN_DAYS) {
-			const toMark = [];
-			for (const type of types) {
-				if (EMPTY_TYPE_PROTECTED.includes(type)) continue;
-				if ((failedByType.get(type) ?? 0) > 0) continue;
-				if ((skippedByType.get(type) ?? 0) > 0 || (rowsByType.get(type) ?? 0) > 0) continue;
-				if ((await store.engine.listLive({
-					...scope,
-					searchType: type
-				})).some((entry) => entry.rowCount > 0)) continue;
-				toMark.push(type);
-			}
-			if (toMark.length > 0) {
-				await emptyTypesStore.mark({
-					userId: store.userId,
-					siteId
-				}, toMark);
-				if (!quiet) logger.info(`Marked empty for future syncs: ${toMark.join(", ")} (0 rows across ${dates.length} days; pass --force-types to re-probe).`);
-			}
+	}
+	const noRollups = opts.args.rollups === false;
+	let rollupError;
+	const anyRowsSynced = Object.values(totals).some((t) => t.rows > 0);
+	if (!noRollups && anyRowsSynced) {
+		if (!quiet) logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)…`);
+		const rollupStart = Date.now();
+		const results = await rebuildRollups({
+			engine: {
+				runSQL: (sqlOpts) => store.engine.runSQL(sqlOpts),
+				listPartitions: async ({ ctx, table, searchType }) => {
+					return (await store.engine.listLive({
+						userId: ctx.userId,
+						...ctx.siteId !== void 0 ? { siteId: ctx.siteId } : {},
+						table,
+						...searchType !== void 0 ? { searchType } : {}
+					})).map((e) => ({
+						partition: e.partition,
+						bytes: e.bytes
+					}));
+				}
+			},
+			dataSource: store.dataSource,
+			ctx: scope,
+			defs: DEFAULT_ROLLUPS
+		}).catch((err) => {
+			rollupError = err.message;
+			logger.warn(`Rollup rebuild failed: ${err.message}`);
+			return [];
+		});
+		if (!quiet && results.length > 0) {
+			const kb = results.reduce((a, r) => a + r.bytes, 0) / 1024;
+			const ms = Date.now() - rollupStart;
+			logger.success(`Rebuilt ${results.length} rollup(s) in ${ms}ms — ${kb.toFixed(1)} KB`);
 		}
-		if (forceTypes && emptyTypesDoc.emptyTypes.length > 0) {
-			const toClear = [];
-			for (const type of types) if (emptyTypesDoc.emptyTypes.includes(type) && (rowsByType.get(type) ?? 0) > 0) toClear.push(type);
-			if (toClear.length > 0) {
-				await emptyTypesStore.clear({
-					userId: store.userId,
-					siteId
-				}, toClear);
-				if (!quiet) logger.info(`Cleared empty markers for: ${toClear.join(", ")} (re-probe found data).`);
+	}
+	const entities = await syncEntities({
+		store,
+		client,
+		siteUrl,
+		ledger: opts.ledger,
+		sitemaps: opts.args.sitemaps !== false,
+		inspections: opts.args.inspections !== false,
+		inspectLimit: opts.inspectLimit,
+		quiet
+	});
+	await opts.ledger.flush();
+	const coverage = await readStoreCoverage({
+		store,
+		site: siteUrl,
+		inspectLimit: opts.inspectLimit,
+		run: { kind: "none" }
+	});
+	if (!quiet) {
+		for (const line of renderCoverage(coverage)) console.log(`  ${line}`);
+		console.log();
+	}
+	const status = anyFailed || rollupError ? "failed" : control.stop ? "partial" : "completed";
+	return {
+		status,
+		...control.stop ? { stop: control.stop } : {},
+		report: await report(status, totals, {
+			...control.stop ? { stopped: control.stop } : {},
+			...rollupError ? { rollupError } : {},
+			...entities,
+			coverage
+		})
+	};
+}
+function failureReason(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+function clockTime(at) {
+	return new Date(at).toLocaleString("en-US", {
+		hour: "numeric",
+		minute: "2-digit",
+		month: "short",
+		day: "numeric"
+	});
+}
+async function syncEntities(opts) {
+	const { store, client, siteUrl, quiet } = opts;
+	const ctx = {
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl)
+	};
+	const now = () => /* @__PURE__ */ new Date();
+	let sitemaps = { _tag: "disabled" };
+	if (opts.sitemaps) {
+		if (!quiet) logger.info("Saving sitemaps…");
+		sitemaps = await syncSitemaps({
+			client,
+			dataSource: store.dataSource,
+			ctx,
+			siteUrl,
+			now,
+			generationId: randomUUID,
+			loadFeed: async (url) => {
+				const loaded = await loadSitemapUrls(url, {
+					maxUrls: 5e5,
+					maxDocuments: 1e3
+				});
+				return loaded._tag === "ok" ? {
+					_tag: "ok",
+					entries: loaded.value.entries,
+					complete: loaded.value.complete
+				} : loaded;
 			}
+		}).catch((error) => ({
+			_tag: "failed",
+			reason: failureReason(error)
+		}));
+		if (sitemaps._tag === "failed") logger.warn(`Sitemaps not saved: ${sitemaps.reason}`);
+		else if (sitemaps._tag === "list_only") logger.warn(`Saved ${sitemaps.sitemaps} sitemap(s). Kept the stored sitemap URLs: ${sitemaps.reason}`);
+		else if (!quiet) logger.success(`Saved ${sitemaps.sitemaps} sitemap(s) with ${sitemaps.urls.toLocaleString()} URL(s)`);
+		if (sitemaps._tag === "saved" || sitemaps._tag === "list_only") {
+			for (const feed of sitemaps.feeds) if (feed._tag === "unreachable") logger.warn(`  ${feed.path}: ${feed.message}`);
+			else if (!feed.complete) logger.warn(`  ${feed.path}: read stopped early; saved ${feed.urls.toLocaleString()} URL(s)`);
 		}
-		const noRollups = Boolean(args["no-rollups"]);
-		let rollupError;
-		const anyRowsSynced = Object.values(totals).some((t) => t.rows > 0);
-		if (!noRollups && anyRowsSynced) {
-			if (!quiet) logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)…`);
-			const rollupStart = Date.now();
-			const results = await rebuildRollups({
-				engine: {
-					runSQL: (opts) => store.engine.runSQL(opts),
-					listPartitions: async ({ ctx, table, searchType }) => {
-						return (await store.engine.listLive({
-							userId: ctx.userId,
-							...ctx.siteId !== void 0 ? { siteId: ctx.siteId } : {},
-							table,
-							...searchType !== void 0 ? { searchType } : {}
-						})).map((e) => ({
-							partition: e.partition,
-							bytes: e.bytes
-						}));
-					}
-				},
+	}
+	let inspections = { _tag: "disabled" };
+	if (opts.inspections && opts.inspectLimit > 0) {
+		inspections = await (async () => {
+			const candidates = await inspectionCandidates(store, siteUrl);
+			const progress = createProgressTracker(Math.min(opts.inspectLimit, candidates.length), quiet);
+			return await syncInspections({
+				client,
 				dataSource: store.dataSource,
-				ctx: {
-					userId: store.userId,
-					siteId
-				},
-				defs: DEFAULT_ROLLUPS
-			}).catch((err) => {
-				rollupError = err.message;
-				logger.warn(`Rollup rebuild failed: ${err.message}`);
-				return [];
-			});
-			if (!quiet && results.length > 0) {
-				const kb = results.reduce((a, r) => a + r.bytes, 0) / 1024;
-				const ms = Date.now() - rollupStart;
-				logger.success(`Rebuilt ${results.length} rollup(s) in ${ms}ms — ${kb.toFixed(1)} KB`);
+				ctx,
+				siteUrl,
+				candidates,
+				limit: opts.inspectLimit,
+				concurrency: INSPECT_CONCURRENCY,
+				ledger: opts.ledger,
+				now,
+				onProgress: () => progress.tick("inspect")
+			}).finally(() => progress.done());
+		})().catch((error) => ({
+			_tag: "failed",
+			reason: failureReason(error)
+		}));
+		if (inspections._tag === "failed") logger.warn(`URL Inspection not saved: ${inspections.reason}`);
+		else if (inspections._tag === "quota_exhausted") logger.warn(`URL Inspection paused until ${clockTime(inspections.resetsAt)}: ${inspections.reason}`);
+		else if (inspections._tag === "inspected") {
+			if (!quiet) logger.success(`Inspected ${inspections.inspected} URL(s); ${inspections.deferred} due URL(s) left for later runs`);
+			if (inspections.stopped) logger.warn(`URL Inspection paused until ${clockTime(inspections.stopped.resetsAt)}: ${inspections.stopped.reason}`);
+			if (inspections.failed > 0) {
+				logger.warn(`${inspections.failed} URL Inspection call(s) failed:`);
+				for (const failure of inspections.failures.slice(0, 5)) logger.warn(`  ${failure.url}: ${failure.error}`);
 			}
-		}
-		await printCompletion(anyFailed || rollupError ? "failed" : "completed", totals, void 0, rollupError);
-		if (anyFailed || rollupError) process.exit(1);
+		} else if (!quiet) logger.info(inspections.quotaLeft === 0 ? "URL Inspection quota is used up for today" : "No URLs are due for URL Inspection");
 	}
-});
-function isKnownTable(name) {
-	return allTables().includes(name);
+	return {
+		sitemaps,
+		inspections
+	};
 }
-function isKnownSearchType(name) {
-	return ALL_SEARCH_TYPES.includes(name);
+function gatedClient(client, gate) {
+	const query = client.searchAnalytics.query;
+	return {
+		...client,
+		searchAnalytics: {
+			...client.searchAnalytics,
+			query: async (...args) => {
+				const [siteUrl] = args;
+				if (gate.maxCalls !== void 0 && gate.calls.made >= gate.maxCalls) throw stopError({
+					kind: "budget",
+					maxCalls: gate.maxCalls
+				});
+				const decision = gate.ledger.reserve("searchAnalytics", siteUrl, 1);
+				if (decision.kind === "exhausted") throw stopError({
+					kind: "quota",
+					reason: decision.reason,
+					resetsAt: decision.resetsAt
+				});
+				gate.calls.made++;
+				return pacer(gate.pacer, () => query(...args)).catch((error) => {
+					const refusal = parseQuotaRefusal(error);
+					if (!refusal) throw error;
+					gate.ledger.record("searchAnalytics", siteUrl, {
+						kind: "refused",
+						reason: refusal
+					});
+					throw stopError({
+						kind: "quota",
+						reason: refusal,
+						resetsAt: gate.ledger.status("searchAnalytics", siteUrl).blocked?.until ?? Date.now()
+					});
+				});
+			}
+		}
+	};
 }
-async function printSyncStatus(resolved, siteFilter, asJson) {
+function pacer(requestPacer, task) {
+	return requestPacer.run(task);
+}
+async function resolveStatusSite(ctx, input) {
+	const inStore = await ctx.matchSite(input, { scope: "store" });
+	if (inStore.kind === "resolved") return inStore.siteUrl;
+	const record = await readSyncRun(ctx.dataDir);
+	const inRun = record ? resolveSiteInput(input, record.sites.map((siteUrl) => ({ siteUrl }))) : void 0;
+	if (inRun?.kind === "resolved") return inRun.siteUrl;
+	throw new Error(formatSiteResolution(inStore, "store"));
+}
+async function printSyncStatus(resolved, siteFilter, asJson, inspectLimit) {
 	const store = createLocalStore({ dataDir: resolved.dataDir });
 	const siteId = siteFilter ? store.siteIdFor(siteFilter) : void 0;
-	const watermarks = await store.engine.getWatermarks({
+	const run = syncRunStatus(await readSyncRun(store.dataDir), {
+		now: Date.now(),
+		isAlive: isProcessAlive
+	});
+	const siteMap = await readSiteMap(store.dataDir, store.userId);
+	const siteLabel = (id) => id ? `@${siteUrlForId(siteMap, id)}` : "";
+	const watermarks = (await store.engine.getWatermarks({
 		userId: store.userId,
 		siteId
-	});
+	})).map((w) => ({
+		...w,
+		siteUrl: w.siteId ? siteUrlForId(siteMap, w.siteId) : null
+	}));
 	const states = await store.engine.getSyncStates({
 		userId: store.userId,
 		siteId
 	});
 	const failed = states.filter((s) => s.state === "failed");
 	const inflight = states.filter((s) => s.state === "inflight");
+	const staleInflight = run.kind === "running" ? [] : inflight;
+	const bySite = /* @__PURE__ */ new Map();
+	for (const state of states) {
+		const key = state.siteId ?? "";
+		bySite.set(key, [...bySite.get(key) ?? [], state]);
+	}
+	const latest = getLatestGscDate();
+	const gaps = [...bySite].flatMap(([id, siteStates]) => analyticsCoverage({
+		states: siteStates,
+		latest,
+		floor: getOldestGscDate(),
+		today: getPstDate()
+	}).jobs.map((job) => ({
+		siteId: id,
+		siteUrl: id ? siteUrlForId(siteMap, id) : null,
+		table: job.table,
+		searchType: job.searchType,
+		from: job.from,
+		to: job.to,
+		done: job.coverage.kind === "complete" ? job.coverage.done : job.coverage.kind === "partial" ? job.coverage.done : 0,
+		missing: job.coverage.kind === "partial" ? job.coverage.pending : 0,
+		failed: job.coverage.kind === "partial" ? job.coverage.failed : 0
+	})));
+	const coverageSites = siteFilter ? [siteFilter] : [.../* @__PURE__ */ new Set([...run.kind === "none" ? [] : run.record.sites, ...resolved.config.defaultSite ? [resolved.config.defaultSite] : []])];
+	const coverages = [];
+	for (const site of coverageSites) coverages.push(await readStoreCoverage({
+		store,
+		site,
+		inspectLimit,
+		run
+	}));
 	if (asJson) {
 		console.log(JSON.stringify({
 			dataDir: store.dataDir,
 			siteFilter: siteFilter ?? null,
+			run,
+			gaps,
+			coverage: coverages,
 			watermarks,
 			failed,
-			inflight
+			inflight,
+			staleInflight: staleInflight.length
 		}, null, 2));
 		return;
 	}
@@ -513,31 +983,39 @@ async function printSyncStatus(resolved, siteFilter, asJson) {
 	console.log(`  \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
 	if (siteFilter) console.log(`  \x1B[90mSite: ${siteFilter}\x1B[0m`);
 	console.log();
-	if (watermarks.length === 0) {
-		console.log(`  No sync watermarks. Run \`gscdump sync\` to ingest data.`);
+	if (run.kind === "running") console.log(`  Sync running: ${run.record.done}/${run.record.planned} days${run.record.site ? ` for ${run.record.site}` : ""}, pid ${run.record.pid}, started ${formatAge(run.record.startedAt)}.`);
+	else if (run.kind === "stale") console.log(`  \x1B[33mThe last sync stopped without finishing (pid ${run.record.pid}, ${run.reason === "process-gone" ? "process gone" : "no heartbeat"}). The next sync retries its dates.\x1B[0m`);
+	if (gaps.length === 0 && watermarks.length === 0) {
+		console.log(`  No synced data. Run \`gscdump sync\` to fill the Store.`);
 		console.log();
 		return;
 	}
-	console.log(`  \x1B[1mWatermarks:\x1B[0m`);
-	const sorted = [...watermarks].sort((a, b) => {
-		if (a.table !== b.table) return a.table.localeCompare(b.table);
-		return (a.siteId ?? "").localeCompare(b.siteId ?? "");
-	});
-	for (const w of sorted) {
-		const scope = w.siteId ? `${w.table}@${w.siteId}` : w.table;
-		console.log(`  ${scope.padEnd(28)} \x1B[36m${w.oldestDateSynced}\x1B[0m → \x1B[36m${w.newestDateSynced}\x1B[0m  \x1B[90m(last ${formatAge(w.lastSyncAt)})\x1B[0m`);
+	console.log(`  \x1B[1mTables:\x1B[0m`);
+	for (const gap of gaps) {
+		const label = `${gap.searchType === "web" ? gap.table : `${gap.table}/${gap.searchType}`}${siteFilter ? "" : siteLabel(gap.siteId)}`;
+		const parts = [`${gap.done} done`];
+		if (gap.missing > 0) parts.push(`${gap.missing} missing`);
+		if (gap.failed > 0) parts.push(`\x1B[31m${gap.failed} failed\x1B[0m`);
+		console.log(`  ${label.padEnd(34)} \x1B[36m${gap.from}\x1B[0m → \x1B[36m${gap.to}\x1B[0m  ${parts.join(", ")}`);
 	}
-	if (inflight.length > 0) {
+	if (staleInflight.length > 0) {
 		console.log();
-		console.log(`  \x1B[33m${inflight.length} inflight:\x1B[0m`);
-		for (const s of inflight) console.log(`    ${s.table}${s.siteId ? `@${s.siteId}` : ""} ${s.date} (attempt ${s.attempts}, started ${formatAge(s.updatedAt)})`);
+		console.log(`  \x1B[33m${staleInflight.length} date(s) stale from a stopped sync. The next sync retries them.\x1B[0m`);
 	}
 	if (failed.length > 0) {
 		console.log();
 		console.log(`  \x1B[31m${failed.length} failed:\x1B[0m`);
-		for (const s of failed) console.log(`    ${s.table}${s.siteId ? `@${s.siteId}` : ""} ${s.date}: ${s.error ?? "unknown"}`);
+		for (const s of failed.slice(0, 20)) console.log(`    ${s.table}${siteLabel(s.siteId)} ${s.date}: ${s.error ?? "unknown"}`);
+		if (failed.length > 20) console.log(`    and ${failed.length - 20} more`);
+	}
+	for (const coverage of coverages) {
+		console.log();
+		console.log(`  \x1B[1m${coverage.site}\x1B[0m`);
+		for (const line of renderCoverage(coverage)) console.log(`  ${line}`);
+	}
+	if (coverages.length === 0) {
 		console.log();
-		console.log(`  Re-run \`gscdump sync --force\` to retry failed dates.`);
+		console.log(`  Pass --site to see inspection and sitemap coverage.`);
 	}
 	console.log();
 }