@gscdump/cli 1.5.4 → 2.0.1

package/README.md

@@ -47,14 +47,14 @@ gscdump mcp
 | `sites [--owner-only] [--with-sitemaps]` | List available GSC sites |
 | `sites add <url>` / `sites delete <url> [--yes]` | Register / remove a property in Search Console (add registers in unverified state) |
 | `sites verify-token <url> [--method]` / `sites verify <url> [--method]` | Get a verification token, then trigger ownership verification (META/FILE/DNS_TXT/DNS_CNAME/ANALYTICS/TAG_MANAGER) |
-| `sitemaps` | Sitemap CRUD (`list [--pending] [--errored]`, `get`, `submit`, `delete`) |
+| `sitemaps` | GSC CRUD, explicit live probes, and hosted canonical reads (`current`, `history`, `membership`, `lastmod`, `export`) |
 | `inspect <url>` / `inspect batch [--concurrency]` | URL inspection (single URL or batch from file/stdin); renders index status, rich results, AMP, mobile usability |
 | `indexing` | Notify Google about URL changes (`submit`, `remove`, `status`, `batch [--concurrency] [--yes]`); supports `--retries` |
 | `sync` | Sync GSC data to the local Parquet store; `--retry-failed`, `--dry-run` |
 | `query` | Run a search analytics query (local store by default; `--live` hits GSC API). Filters: `--query`, `--page`, `--country`, `--device`, `--search-appearance`, `--type`, `--data-state`, `--aggregation-type`. `--explain` previews the request body; `--output -` writes to stdout. |
 | `dump` | Export from the store to a directory (`--format parquet\|json\|ndjson\|csv`, `--tables`, `--all-sites`) |
 | `analyze <tool>` | Run an SEO analyzer against the store (`--live` for row-based against fresh API) |
-| `entities` | Snapshot URL inspections / sitemaps / indexing metadata into the local entity store |
+| `entities` | Snapshot URL inspections and indexing metadata into the local entity store |
 | `store stats` | Show row/byte counts per table and on-disk footprint |
 | `store compact` | Roll daily partitions older than N days into monthly files (`--dry-run`) |
 | `store gc` | Delete orphaned objects past the grace window (`--dry-run`) |

package/dist/commands/entities.mjs

@@ -4,7 +4,7 @@ import process from "node:process";
 import { defineCommand } from "citty";
 import { readFile } from "node:fs/promises";
 import { Buffer } from "node:buffer";
-import { createIndexingMetadataStore, createInspectionStore, createSitemapReadStore, createSitemapStore } from "@gscdump/engine/entities";
+import { createIndexingMetadataStore, createInspectionStore } from "@gscdump/engine/entities";
 const INSPECTION_QPD_PER_PROPERTY = 2e3;
 const INDEXING_NOT_FOUND_RE = /\b404\b|NOT_FOUND/i;
 const INSPECTION_HISTORY_LOOKBACK_MONTHS = 24;
@@ -27,390 +27,250 @@ async function readUrlList(opts) {
 	for await (const chunk of process.stdin) chunks.push(chunk);
 	return Buffer.concat(chunks).toString("utf8").split("\n").map((l) => l.trim()).filter(Boolean);
 }
-const inspectSubCommand = defineCommand({
-	meta: {
-		name: "inspect",
-		description: "Run URL Inspection for a list of URLs and persist results to the local entity store"
-	},
-	args: {
-		site: {
-			type: "string",
-			alias: "s",
-			description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
-		},
-		file: {
-			type: "string",
-			alias: "f",
-			description: "Path to a file with one URL per line. If omitted, reads from stdin."
-		},
-		limit: {
-			type: "string",
-			description: `Max URLs to inspect this run (default: ${INSPECTION_QPD_PER_PROPERTY}, the per-property GSC daily quota)`
-		},
-		concurrency: {
-			type: "string",
-			alias: "c",
-			default: "4",
-			description: "Concurrent in-flight inspect calls (default: 4)"
-		},
-		...OUTPUT_ARGS
-	},
-	async run({ args }) {
-		const { json, quiet } = applyOutputMode(args);
-		const ctx = await createCommandContext({
-			needsAuth: true,
-			needsStore: true
-		});
-		const client = ctx.client;
-		const store = ctx.store;
-		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
-		const limit = args.limit ? Number.parseInt(String(args.limit), 10) : INSPECTION_QPD_PER_PROPERTY;
-		const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
-		const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
-		if (urls.length === 0) {
-			logger.warn("No URLs to inspect.");
-			return;
-		}
-		if (urls.length === limit && limit < INSPECTION_QPD_PER_PROPERTY) logger.info(`Capping at --limit ${limit}`);
-		if (urls.length === INSPECTION_QPD_PER_PROPERTY) logger.info(`Hit per-property daily inspection quota (${INSPECTION_QPD_PER_PROPERTY}); remaining URLs will be queued for tomorrow.`);
-		const inspector = createInspectionStore({ dataSource: store.dataSource });
-		let completed = 0;
-		let failed = 0;
-		const records = [];
-		const failures = [];
-		await runWithConcurrency(urls, concurrency, async (url) => {
-			const result = await client.inspect(siteUrl, url).catch((err) => err);
-			if (result instanceof Error) {
-				failed++;
-				failures.push({
-					url,
-					error: result.message
-				});
-			} else {
-				const ix = result.inspectionResult;
-				const indexStatus = ix?.indexStatusResult;
-				records.push({
-					url,
-					inspectedAt: (/* @__PURE__ */ new Date()).toISOString(),
-					indexStatus: indexStatus?.verdict ?? void 0,
-					lastCrawlTime: indexStatus?.lastCrawlTime ?? void 0,
-					googleCanonical: indexStatus?.googleCanonical ?? void 0,
-					userCanonical: indexStatus?.userCanonical ?? void 0,
-					coverageState: indexStatus?.coverageState ?? void 0,
-					robotsTxtState: indexStatus?.robotsTxtState ?? void 0,
-					indexingState: indexStatus?.indexingState ?? void 0,
-					pageFetchState: indexStatus?.pageFetchState ?? void 0,
-					mobileUsabilityVerdict: ix?.mobileUsabilityResult?.verdict ?? void 0,
-					richResultsVerdict: ix?.richResultsResult?.verdict ?? void 0,
-					raw: ix
-				});
-			}
-			completed++;
-			if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, `${url.slice(0, 60)}`)}`);
-		});
-		if (!quiet) process.stdout.write("\n");
-		await inspector.appendHistory({
-			userId: store.userId,
-			siteId: store.siteIdFor(siteUrl)
-		}, records);
-		if (json) console.log(JSON.stringify({
-			site: siteUrl,
-			inspected: records.length,
-			failed,
-			failures,
-			records
-		}, null, 2));
-		else if (!quiet) {
-			logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
-			if (failed > 0) {
-				logger.warn(`${failed} failed:`);
-				for (const f of failures.slice(0, 5)) console.log(`  ${f.url}: ${f.error}`);
-				if (failures.length > 5) console.log(`  ... and ${failures.length - 5} more`);
-			}
-		}
-		if (failed > 0) process.exit(1);
-	}
-});
-const showSubCommand = defineCommand({
-	meta: {
-		name: "show",
-		description: "Print the latest inspection record for a URL from the local entity store"
-	},
-	args: {
-		...OUTPUT_ARGS,
-		site: {
-			type: "string",
-			alias: "s",
-			description: "Site URL (defaults to config.defaultSite or prompt)"
-		},
-		url: {
-			type: "positional",
-			required: true,
-			description: "URL to look up"
-		}
-	},
-	async run({ args }) {
-		const { json } = applyOutputMode(args);
-		const ctx = await createCommandContext({
-			needsAuth: true,
-			needsStore: true
-		});
-		const store = ctx.store;
-		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
-		const record = await findLatestInspection(createInspectionStore({ dataSource: store.dataSource }), {
-			userId: store.userId,
-			siteId: store.siteIdFor(siteUrl)
-		}, String(args.url));
-		if (!record) {
-			logger.warn(`No inspection record for ${args.url}`);
-			process.exit(1);
-		}
-		if (json) {
-			console.log(JSON.stringify(record, null, 2));
-			return;
-		}
-		console.log();
-		console.log(`  \x1B[1m${record.url}\x1B[0m`);
-		console.log(`  Inspected:    ${record.inspectedAt}`);
-		if (record.indexStatus) console.log(`  Index:        ${record.indexStatus}`);
-		if (record.lastCrawlTime) console.log(`  Last crawl:   ${record.lastCrawlTime}`);
-		if (record.googleCanonical) console.log(`  Canonical:    ${record.googleCanonical}`);
-		if (record.coverageState) console.log(`  Coverage:     ${record.coverageState}`);
-		if (record.mobileUsabilityVerdict) console.log(`  Mobile:       ${record.mobileUsabilityVerdict}`);
-		if (record.richResultsVerdict) console.log(`  Rich results: ${record.richResultsVerdict}`);
-		console.log();
-	}
-});
-const sitemapsSnapshotSubCommand = defineCommand({
-	meta: {
-		name: "snapshot",
-		description: "Fetch current sitemap state from GSC and persist to the local entity store"
-	},
-	args: {
-		...OUTPUT_ARGS,
-		site: {
-			type: "string",
-			alias: "s",
-			description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
-		}
-	},
-	async run({ args }) {
-		const { json, quiet } = applyOutputMode(args);
-		const ctx = await createCommandContext({
-			needsAuth: true,
-			needsStore: true
-		});
-		const client = ctx.client;
-		const store = ctx.store;
-		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
-		const apiSitemaps = await client.sitemaps.list(siteUrl);
-		const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
-		const records = apiSitemaps.filter((s) => typeof s.path === "string").map((s) => ({
-			path: s.path,
-			capturedAt,
-			lastDownloaded: s.lastDownloaded ?? void 0,
-			lastSubmitted: s.lastSubmitted ?? void 0,
-			type: s.type ?? void 0,
-			isPending: s.isPending ?? void 0,
-			isSitemapsIndex: s.isSitemapsIndex ?? void 0,
-			errors: s.errors ?? void 0,
-			warnings: s.warnings ?? void 0,
-			contents: s.contents?.map((c) => ({
-				type: c.type ?? void 0,
-				submitted: c.submitted ?? void 0,
-				indexed: c.indexed ?? void 0
-			})),
-			raw: s
-		}));
-		await createSitemapStore({
-			dataSource: store.dataSource,
-			withMutation: store.withSitemapMutation
-		}).writeSnapshot({
-			userId: store.userId,
-			siteId: store.siteIdFor(siteUrl)
-		}, records);
-		if (json) {
-			console.log(JSON.stringify({
-				site: siteUrl,
-				capturedAt,
-				records
-			}, null, 2));
-			return;
-		}
-		if (!quiet) {
-			logger.success(`Captured ${records.length} sitemap(s) for ${siteUrl}`);
-			for (const r of records) {
-				const errors = r.errors && r.errors !== "0" ? ` \x1B[31merr=${r.errors}\x1B[0m` : "";
-				const warnings = r.warnings && r.warnings !== "0" ? ` \x1B[33mwarn=${r.warnings}\x1B[0m` : "";
-				const downloaded = r.lastDownloaded ? ` last=${r.lastDownloaded}` : "";
-				console.log(`  ${r.path}${downloaded}${errors}${warnings}`);
-			}
-		}
-	}
-});
-const sitemapsShowSubCommand = defineCommand({
-	meta: {
-		name: "show",
-		description: "Print the latest captured sitemap state for a feedpath"
-	},
-	args: {
-		...OUTPUT_ARGS,
-		site: {
-			type: "string",
-			alias: "s",
-			description: "Site URL (defaults to config.defaultSite or prompt)"
-		},
-		path: {
-			type: "positional",
-			required: true,
-			description: "Sitemap path (feedpath)"
-		}
-	},
-	async run({ args }) {
-		const { json } = applyOutputMode(args);
-		const ctx = await createCommandContext({
-			needsAuth: true,
-			needsStore: true
-		});
-		const store = ctx.store;
-		const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
-		const record = await createSitemapReadStore({ dataSource: store.dataSource }).getLatest({
-			userId: store.userId,
-			siteId: store.siteIdFor(siteUrl)
-		}, String(args.path));
-		if (!record) {
-			logger.warn(`No sitemap record for ${args.path}`);
-			process.exit(1);
-		}
-		if (json) {
-			console.log(JSON.stringify(record, null, 2));
-			return;
-		}
-		console.log();
-		console.log(`  \x1B[1m${record.path}\x1B[0m`);
-		console.log(`  Captured:     ${record.capturedAt}`);
-		if (record.lastDownloaded) console.log(`  Downloaded:   ${record.lastDownloaded}`);
-		if (record.lastSubmitted) console.log(`  Submitted:    ${record.lastSubmitted}`);
-		if (record.type) console.log(`  Type:         ${record.type}`);
-		if (record.errors) console.log(`  Errors:       ${record.errors}`);
-		if (record.warnings) console.log(`  Warnings:     ${record.warnings}`);
-		if (record.contents?.length) {
-			console.log(`  Contents:`);
-			for (const c of record.contents) {
-				const bits = [
-					c.type,
-					c.submitted && `submitted=${c.submitted}`,
-					c.indexed && `indexed=${c.indexed}`
-				].filter(Boolean).join("  ");
-				console.log(`    ${bits}`);
-			}
-		}
-		console.log();
-	}
-});
-const indexingSubCommand = defineCommand({
+const entitiesCommand = defineCommand({
 	meta: {
-		name: "indexing",
-		description: "Snapshot Indexing API metadata per URL"
+		name: "entities",
+		description: "Manage local entity snapshots (URL inspections and indexing metadata)"
 	},
-	subCommands: { snapshot: defineCommand({
-		meta: {
-			name: "snapshot",
-			description: "Fetch Indexing API metadata (latest update/remove per URL) and persist to the local entity store"
-		},
-		args: {
-			site: {
-				type: "string",
-				alias: "s",
-				description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
-			},
-			file: {
-				type: "string",
-				alias: "f",
-				description: "Path to a file with one URL per line. If omitted, reads from stdin."
+	subCommands: {
+		inspect: defineCommand({
+			meta: {
+				name: "inspect",
+				description: "Run URL Inspection for a list of URLs and persist results to the local entity store"
 			},
-			concurrency: {
-				type: "string",
-				alias: "c",
-				default: "4",
-				description: "Concurrent in-flight getMetadata calls (default: 4)"
+			args: {
+				site: {
+					type: "string",
+					alias: "s",
+					description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
+				},
+				file: {
+					type: "string",
+					alias: "f",
+					description: "Path to a file with one URL per line. If omitted, reads from stdin."
+				},
+				limit: {
+					type: "string",
+					description: `Max URLs to inspect this run (default: ${INSPECTION_QPD_PER_PROPERTY}, the per-property GSC daily quota)`
+				},
+				concurrency: {
+					type: "string",
+					alias: "c",
+					default: "4",
+					description: "Concurrent in-flight inspect calls (default: 4)"
+				},
+				...OUTPUT_ARGS
 			},
-			...OUTPUT_ARGS
-		},
-		async run({ args }) {
-			const { quiet } = applyOutputMode(args);
-			const ctx = await createCommandContext({
-				needsAuth: true,
-				needsStore: true
-			});
-			const client = ctx.client;
-			const store = ctx.store;
-			const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
-			const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
-			const urls = await readUrlList({ file: args.file ? String(args.file) : void 0 });
-			if (urls.length === 0) {
-				logger.warn("No URLs to fetch metadata for.");
-				return;
-			}
-			const records = [];
-			const failures = [];
-			let completed = 0;
-			await runWithConcurrency(urls, concurrency, async (url) => {
-				const result = await client.indexing.getMetadata(url).catch((err) => err);
-				if (result instanceof Error) if (INDEXING_NOT_FOUND_RE.test(result.message)) records.push({
-					url,
-					capturedAt: (/* @__PURE__ */ new Date()).toISOString()
-				});
-				else failures.push({
-					url,
-					error: result.message
+			async run({ args }) {
+				const { json, quiet } = applyOutputMode(args);
+				const ctx = await createCommandContext({
+					needsAuth: true,
+					needsStore: true
 				});
-				else records.push({
-					url,
-					capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
-					latestUpdateAt: result.latestUpdate?.notifyTime ?? void 0,
-					latestRemoveAt: result.latestRemove?.notifyTime ?? void 0,
-					raw: result
+				const client = ctx.client;
+				const store = ctx.store;
+				const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+				const limit = args.limit ? Number.parseInt(String(args.limit), 10) : INSPECTION_QPD_PER_PROPERTY;
+				const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
+				const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
+				if (urls.length === 0) {
+					logger.warn("No URLs to inspect.");
+					return;
+				}
+				if (urls.length === limit && limit < INSPECTION_QPD_PER_PROPERTY) logger.info(`Capping at --limit ${limit}`);
+				if (urls.length === INSPECTION_QPD_PER_PROPERTY) logger.info(`Hit per-property daily inspection quota (${INSPECTION_QPD_PER_PROPERTY}); remaining URLs will be queued for tomorrow.`);
+				const inspector = createInspectionStore({ dataSource: store.dataSource });
+				let completed = 0;
+				let failed = 0;
+				const records = [];
+				const failures = [];
+				await runWithConcurrency(urls, concurrency, async (url) => {
+					const result = await client.inspect(siteUrl, url).catch((err) => err);
+					if (result instanceof Error) {
+						failed++;
+						failures.push({
+							url,
+							error: result.message
+						});
+					} else {
+						const ix = result.inspectionResult;
+						const indexStatus = ix?.indexStatusResult;
+						records.push({
+							url,
+							inspectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+							indexStatus: indexStatus?.verdict ?? void 0,
+							lastCrawlTime: indexStatus?.lastCrawlTime ?? void 0,
+							googleCanonical: indexStatus?.googleCanonical ?? void 0,
+							userCanonical: indexStatus?.userCanonical ?? void 0,
+							coverageState: indexStatus?.coverageState ?? void 0,
+							robotsTxtState: indexStatus?.robotsTxtState ?? void 0,
+							indexingState: indexStatus?.indexingState ?? void 0,
+							pageFetchState: indexStatus?.pageFetchState ?? void 0,
+							mobileUsabilityVerdict: ix?.mobileUsabilityResult?.verdict ?? void 0,
+							richResultsVerdict: ix?.richResultsResult?.verdict ?? void 0,
+							raw: ix
+						});
+					}
+					completed++;
+					if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, `${url.slice(0, 60)}`)}`);
 				});
-				completed++;
-				if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, url.slice(0, 60))}`);
-			});
-			if (!quiet) process.stdout.write("\n");
-			await createIndexingMetadataStore({ dataSource: store.dataSource }).writeBatch({
-				userId: store.userId,
-				siteId: store.siteIdFor(siteUrl)
-			}, records);
-			if (!quiet) {
-				logger.success(`Captured metadata for ${records.length}/${urls.length} URL(s)`);
-				if (failures.length > 0) {
-					logger.warn(`${failures.length} failed:`);
-					for (const f of failures.slice(0, 5)) console.log(`  ${f.url}: ${f.error}`);
-					if (failures.length > 5) console.log(`  ... and ${failures.length - 5} more`);
+				if (!quiet) process.stdout.write("\n");
+				await inspector.appendHistory({
+					userId: store.userId,
+					siteId: store.siteIdFor(siteUrl)
+				}, records);
+				if (json) console.log(JSON.stringify({
+					site: siteUrl,
+					inspected: records.length,
+					failed,
+					failures,
+					records
+				}, null, 2));
+				else if (!quiet) {
+					logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
+					if (failed > 0) {
+						logger.warn(`${failed} failed:`);
+						for (const f of failures.slice(0, 5)) console.log(`  ${f.url}: ${f.error}`);
+						if (failures.length > 5) console.log(`  ... and ${failures.length - 5} more`);
+					}
 				}
+				if (failed > 0) process.exit(1);
 			}
-			if (failures.length > 0) process.exit(1);
-		}
-	}) }
-});
-const entitiesCommand = defineCommand({
-	meta: {
-		name: "entities",
-		description: "Manage local entity snapshots (URL inspections, sitemaps, indexing metadata)"
-	},
-	subCommands: {
-		inspect: inspectSubCommand,
-		show: showSubCommand,
-		sitemaps: defineCommand({
+		}),
+		show: defineCommand({
 			meta: {
-				name: "sitemaps",
-				description: "Snapshot and inspect sitemap state per site"
+				name: "show",
+				description: "Print the latest inspection record for a URL from the local entity store"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				site: {
+					type: "string",
+					alias: "s",
+					description: "Site URL (defaults to config.defaultSite or prompt)"
+				},
+				url: {
+					type: "positional",
+					required: true,
+					description: "URL to look up"
+				}
 			},
-			subCommands: {
-				snapshot: sitemapsSnapshotSubCommand,
-				show: sitemapsShowSubCommand
+			async run({ args }) {
+				const { json } = applyOutputMode(args);
+				const ctx = await createCommandContext({
+					needsAuth: true,
+					needsStore: true
+				});
+				const store = ctx.store;
+				const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+				const record = await findLatestInspection(createInspectionStore({ dataSource: store.dataSource }), {
+					userId: store.userId,
+					siteId: store.siteIdFor(siteUrl)
+				}, String(args.url));
+				if (!record) {
+					logger.warn(`No inspection record for ${args.url}`);
+					process.exit(1);
+				}
+				if (json) {
+					console.log(JSON.stringify(record, null, 2));
+					return;
+				}
+				console.log();
+				console.log(`  \x1B[1m${record.url}\x1B[0m`);
+				console.log(`  Inspected:    ${record.inspectedAt}`);
+				if (record.indexStatus) console.log(`  Index:        ${record.indexStatus}`);
+				if (record.lastCrawlTime) console.log(`  Last crawl:   ${record.lastCrawlTime}`);
+				if (record.googleCanonical) console.log(`  Canonical:    ${record.googleCanonical}`);
+				if (record.coverageState) console.log(`  Coverage:     ${record.coverageState}`);
+				if (record.mobileUsabilityVerdict) console.log(`  Mobile:       ${record.mobileUsabilityVerdict}`);
+				if (record.richResultsVerdict) console.log(`  Rich results: ${record.richResultsVerdict}`);
+				console.log();
 			}
 		}),
-		indexing: indexingSubCommand
+		indexing: defineCommand({
+			meta: {
+				name: "indexing",
+				description: "Snapshot Indexing API metadata per URL"
+			},
+			subCommands: { snapshot: defineCommand({
+				meta: {
+					name: "snapshot",
+					description: "Fetch Indexing API metadata (latest update/remove per URL) and persist to the local entity store"
+				},
+				args: {
+					site: {
+						type: "string",
+						alias: "s",
+						description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
+					},
+					file: {
+						type: "string",
+						alias: "f",
+						description: "Path to a file with one URL per line. If omitted, reads from stdin."
+					},
+					concurrency: {
+						type: "string",
+						alias: "c",
+						default: "4",
+						description: "Concurrent in-flight getMetadata calls (default: 4)"
+					},
+					...OUTPUT_ARGS
+				},
+				async run({ args }) {
+					const { quiet } = applyOutputMode(args);
+					const ctx = await createCommandContext({
+						needsAuth: true,
+						needsStore: true
+					});
+					const client = ctx.client;
+					const store = ctx.store;
+					const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
+					const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
+					const urls = await readUrlList({ file: args.file ? String(args.file) : void 0 });
+					if (urls.length === 0) {
+						logger.warn("No URLs to fetch metadata for.");
+						return;
+					}
+					const records = [];
+					const failures = [];
+					let completed = 0;
+					await runWithConcurrency(urls, concurrency, async (url) => {
+						const result = await client.indexing.getMetadata(url).catch((err) => err);
+						if (result instanceof Error) if (INDEXING_NOT_FOUND_RE.test(result.message)) records.push({
+							url,
+							capturedAt: (/* @__PURE__ */ new Date()).toISOString()
+						});
+						else failures.push({
+							url,
+							error: result.message
+						});
+						else records.push({
+							url,
+							capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
+							latestUpdateAt: result.latestUpdate?.notifyTime ?? void 0,
+							latestRemoveAt: result.latestRemove?.notifyTime ?? void 0,
+							raw: result
+						});
+						completed++;
+						if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, url.slice(0, 60))}`);
+					});
+					if (!quiet) process.stdout.write("\n");
+					await createIndexingMetadataStore({ dataSource: store.dataSource }).writeBatch({
+						userId: store.userId,
+						siteId: store.siteIdFor(siteUrl)
+					}, records);
+					if (!quiet) {
+						logger.success(`Captured metadata for ${records.length}/${urls.length} URL(s)`);
+						if (failures.length > 0) {
+							logger.warn(`${failures.length} failed:`);
+							for (const f of failures.slice(0, 5)) console.log(`  ${f.url}: ${f.error}`);
+							if (failures.length > 5) console.log(`  ... and ${failures.length - 5} more`);
+						}
+					}
+					if (failures.length > 0) process.exit(1);
+				}
+			}) }
+		})
 	}
 });
 export { entitiesCommand };

package/dist/commands/sitemaps.mjs

@@ -1,11 +1,32 @@
+import { resolveCliEnvironment } from "../environment.mjs";
 import { OUTPUT_ARGS, applyOutputMode, logger, noSubcommandSelected } from "../utils.mjs";
 import { createCommandContext } from "../context.mjs";
 import { gscErrorHandler } from "../error-handler.mjs";
-import { loadSitemapUrls } from "../sitemap.mjs";
+import { discoverLiveSitemap, loadSitemapUrls } from "../sitemap.mjs";
 import process from "node:process";
 import { defineCommand } from "citty";
 import { fetchSitemap } from "gscdump";
-import { discoverSitemapResult } from "gscdump/sitemap";
+import { createGscdumpV1Client } from "@gscdump/sdk/v1";
+const HOSTED_ARGS = {
+	"api-root": {
+		type: "string",
+		description: "Hosted API root; defaults to GSCDUMP_API_ROOT or https://gscdump.com/api/_gscdump"
+	},
+	"api-key": {
+		type: "string",
+		description: "Hosted API key; defaults to GSCDUMP_API_KEY"
+	}
+};
+function hostedClient(args) {
+	const environment = resolveCliEnvironment().values;
+	const apiRoot = String(args["api-root"] || environment.GSCDUMP_API_ROOT || "https://gscdump.com/api/_gscdump");
+	const apiKey = String(args["api-key"] || environment.GSCDUMP_API_KEY || "");
+	if (!apiKey) throw new Error("Hosted sitemap reads require --api-key or GSCDUMP_API_KEY");
+	return createGscdumpV1Client({
+		apiRoot,
+		credential: apiKey
+	});
+}
 const listCommand = defineCommand({
 	meta: {
 		name: "list",
@@ -197,7 +218,7 @@ const sitemapsCommand = defineCommand({
 			async run({ args }) {
 				const { json } = applyOutputMode(args);
 				const domain = String(args.domain).replace(/^https?:\/\//, "").replace(/\/.*$/, "");
-				const discovery = await discoverSitemapResult(domain);
+				const discovery = await discoverLiveSitemap(domain);
 				const url = discovery._tag === "found" ? discovery.url : null;
 				if (json) {
 					console.log(JSON.stringify({
@@ -208,7 +229,7 @@ const sitemapsCommand = defineCommand({
 					return;
 				}
 				if (discovery._tag === "incomplete") {
-					logger.error(`Sitemap discovery was incomplete: ${discovery.failures[0]?.detail ?? "unknown failure"}`);
+					logger.error(`Sitemap discovery was incomplete: ${discovery.failures[0] ?? "unknown failure"}`);
 					process.exit(1);
 				}
 				if (!url) {
@@ -266,6 +287,191 @@ const sitemapsCommand = defineCommand({
 				if (!complete) logger.warn("Sitemap walk was truncated or a child document failed");
 				for (const u of urls) console.log(u);
 			}
+		}),
+		current: defineCommand({
+			meta: {
+				name: "current",
+				description: "Read the hosted canonical sitemap generation and feed list"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				...HOSTED_ARGS,
+				"site-id": {
+					type: "positional",
+					required: true,
+					description: "Hosted gscdump site id"
+				}
+			},
+			async run({ args }) {
+				const { json } = applyOutputMode(args);
+				const result = await hostedClient(args).getSiteSitemaps({ params: { siteId: String(args["site-id"]) } });
+				if (json) {
+					console.log(JSON.stringify(result.data, null, 2));
+					return;
+				}
+				const generation = result.data.generation;
+				console.log(generation ? `Generation ${generation.id}, observed ${new Date(generation.observedAt).toISOString()}` : "No canonical generation");
+				for (const sitemap of result.data.sitemaps) console.log(`${sitemap.path}\t${sitemap.urlCount}`);
+			}
+		}),
+		history: defineCommand({
+			meta: {
+				name: "history",
+				description: "Read hosted canonical sitemap membership and lastmod changes"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				...HOSTED_ARGS,
+				"site-id": {
+					type: "positional",
+					required: true,
+					description: "Hosted gscdump site id"
+				},
+				"days": {
+					type: "string",
+					description: "History window in days"
+				}
+			},
+			async run({ args }) {
+				const { json } = applyOutputMode(args);
+				const days = args.days ? Number.parseInt(String(args.days), 10) : void 0;
+				const result = await hostedClient(args).getSiteSitemapChanges({
+					params: { siteId: String(args["site-id"]) },
+					query: { ...days ? { days } : {} }
+				});
+				console.log(JSON.stringify(result.data, null, json ? 2 : 0));
+			}
+		}),
+		membership: defineCommand({
+			meta: {
+				name: "membership",
+				description: "Query exact hosted membership evidence for comma-separated URLs"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				...HOSTED_ARGS,
+				"site-id": {
+					type: "positional",
+					required: true,
+					description: "Hosted gscdump site id"
+				},
+				"urls": {
+					type: "string",
+					required: true,
+					description: "Comma-separated exact URLs"
+				},
+				"generation": {
+					type: "string",
+					description: "Pin the query to a generation id"
+				}
+			},
+			async run({ args }) {
+				const { json } = applyOutputMode(args);
+				const urls = String(args.urls).split(",").map((url) => url.trim()).filter(Boolean);
+				const result = await hostedClient(args).querySitemapMembership({
+					params: { siteId: String(args["site-id"]) },
+					body: {
+						urls,
+						...args.generation ? { generationId: String(args.generation) } : {}
+					}
+				});
+				if (json) {
+					console.log(JSON.stringify(result.data, null, 2));
+					return;
+				}
+				for (const evidence of result.data.evidence) {
+					const detail = evidence._tag === "present" ? `${evidence.feedpath}\t${evidence.lastmod ?? ""}` : evidence._tag === "absent" ? String(evidence.observedAt) : evidence.reason;
+					console.log(`${evidence._tag}\t${evidence.url}\t${detail}`);
+				}
+			}
+		}),
+		lastmod: defineCommand({
+			meta: {
+				name: "lastmod",
+				description: "List a generation-pinned hosted page of URL lastmod evidence"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				...HOSTED_ARGS,
+				"site-id": {
+					type: "positional",
+					required: true,
+					description: "Hosted gscdump site id"
+				},
+				"generation": {
+					type: "string",
+					description: "Pin to a generation id"
+				},
+				"feedpath": {
+					type: "string",
+					description: "Restrict to one exact sitemap feed URL"
+				},
+				"cursor": {
+					type: "string",
+					description: "Opaque cursor from the prior page"
+				},
+				"limit": {
+					type: "string",
+					description: "Page size, maximum 10000"
+				}
+			},
+			async run({ args }) {
+				const { json } = applyOutputMode(args);
+				const result = await hostedClient(args).listSitemapUrls({
+					params: { siteId: String(args["site-id"]) },
+					query: {
+						...args.generation ? { generationId: String(args.generation) } : {},
+						...args.feedpath ? { feedpath: String(args.feedpath) } : {},
+						...args.cursor ? { cursor: String(args.cursor) } : {},
+						...args.limit ? { limit: Number.parseInt(String(args.limit), 10) } : {}
+					}
+				});
+				if (json) {
+					console.log(JSON.stringify(result.data, null, 2));
+					return;
+				}
+				for (const item of result.data.items) console.log(`${item.url}\t${item.lastmod ?? ""}\t${item.feedpath}`);
+				if (result.data.page.nextCursor) logger.info(`Next cursor: ${result.data.page.nextCursor}`);
+			}
+		}),
+		export: defineCommand({
+			meta: {
+				name: "export",
+				description: "Get a generation-pinned hosted bulk export descriptor"
+			},
+			args: {
+				...OUTPUT_ARGS,
+				...HOSTED_ARGS,
+				"site-id": {
+					type: "positional",
+					required: true,
+					description: "Hosted gscdump site id"
+				},
+				"generation": {
+					type: "string",
+					description: "Pin to a generation id"
+				},
+				"feedpath": {
+					type: "string",
+					description: "Restrict to one exact sitemap feed URL"
+				}
+			},
+			async run({ args }) {
+				const { json } = applyOutputMode(args);
+				const result = await hostedClient(args).getSitemapExport({
+					params: { siteId: String(args["site-id"]) },
+					query: {
+						...args.generation ? { generationId: String(args.generation) } : {},
+						...args.feedpath ? { feedpath: String(args.feedpath) } : {}
+					}
+				});
+				if (json) {
+					console.log(JSON.stringify(result.data, null, 2));
+					return;
+				}
+				if (result.data.export._tag === "unavailable") throw new Error(`Sitemap export unavailable: ${result.data.export.reason}`);
+				console.log(result.data.export.url);
+			}
 		})
 	},
 	async run({ args }) {
@@ -275,7 +481,12 @@ const sitemapsCommand = defineCommand({
 			"submit",
 			"delete",
 			"discover",
-			"urls"
+			"urls",
+			"current",
+			"history",
+			"membership",
+			"lastmod",
+			"export"
 		])) return;
 		await listCommand.run?.({
 			args,

package/dist/mcp/server/index.mjs

@@ -1,3 +1,4 @@
+import { discoverLiveSitemap } from "../../sitemap.mjs";
 import { diagnostics } from "../handlers/diagnostics.mjs";
 import { batchInspectUrls as batchInspectUrls$1, batchRequestIndexing as batchRequestIndexing$1, getIndexingStatus, requestIndexing as requestIndexing$1 } from "../handlers/indexing.mjs";
 import { listReports, runReportHandler } from "../handlers/reports.mjs";
@@ -5,7 +6,6 @@ import { getSitemap, listSitesWithSitemaps } from "../handlers/sites.mjs";
 import "../handlers/index.mjs";
 import { batchInspectUrlsInput, batchRequestIndexingInput, getIndexingStatusInput, inspectUrlInput, listReportsInput, listSitemapsInput, listSitesInput, requestIndexingInput, runReportInput, sitemapInput } from "../types.mjs";
 import { addSite, deleteSite, getIndexingMetadata, getVerificationToken, getVerifiedSite, googleSearchConsole, listVerifiedSites, runSequentialBatch, unverifySite, verifySite } from "gscdump";
-import { discoverSitemapResult } from "gscdump/sitemap";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
 async function runMcpSearchAnalyticsQuery(client, args) {
@@ -386,7 +386,7 @@ function createGscMcpServer(options) {
 		inputSchema: z.object({ domain: z.string().describe("Domain (e.g., example.com)") }).shape
 	}, async ({ domain }) => {
 		const cleaned = String(domain).replace(/^https?:\/\//, "").replace(/\/.*$/, "");
-		const discovery = await discoverSitemapResult(cleaned);
+		const discovery = await discoverLiveSitemap(cleaned);
 		return { content: [{
 			type: "text",
 			text: JSON.stringify({

package/dist/package.mjs

@@ -1,2 +1,2 @@
-var version = "1.5.4";
+var version = "2.0.1";
 export { version };

package/dist/sitemap.mjs

@@ -1,27 +1,89 @@
-import { walkSitemaps } from "gscdump/sitemap";
-function failureMessage(result) {
-	if (result._tag === "not_found") return "No sitemap document was found";
-	const first = result.failures[0];
-	if (!first) return "Sitemap walk failed before reading a document";
-	const error = first.error;
-	if (error._tag === "network_error") return `${first.url}: ${error.error}`;
-	if (error._tag === "http_error") return `${first.url}: HTTP ${error.status} ${error.statusText}`;
-	if (error._tag === "not_found") return `${first.url}: HTTP ${error.status}`;
-	return `${first.url}: ${error.error._tag}`;
+import { createSitemapReader, parseRobotsSitemaps } from "sitemapd";
+import { createFetchDocumentLoader } from "sitemapd/fetch";
+const COMMON_PATHS = [
+	"/sitemap.xml",
+	"/sitemap_index.xml",
+	"/sitemap-index.xml",
+	"/sitemaps.xml"
+];
+function publicHttpTarget(url) {
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		return {
+			_tag: "deny",
+			reason: "invalid URL"
+		};
+	}
+	return parsed.protocol === "http:" || parsed.protocol === "https:" ? { _tag: "allow" } : {
+		_tag: "deny",
+		reason: "only HTTP(S) sitemap targets are supported"
+	};
+}
+function createLiveReader(fetch = globalThis.fetch) {
+	return createSitemapReader({
+		loadDocument: createFetchDocumentLoader({ fetch }),
+		authorizeTarget: ({ url }) => publicHttpTarget(url)
+	});
+}
+function readFailure(result) {
+	if (result._tag === "not_found") return `${result.url}: HTTP ${result.status}`;
+	return `${result.url}: ${result.code ?? result.reason}: ${result.detail}`;
 }
 async function loadSitemapUrls(url, options = {}) {
-	const result = await walkSitemaps(url, options);
-	if (result._tag !== "ok") return {
+	const result = await createLiveReader().walk(url, options);
+	if (result.documentsRead === 0) return {
 		_tag: "error",
-		message: failureMessage(result)
+		message: result.failures[0] ? readFailure(result.failures[0].result) : "No sitemap document was found"
 	};
 	return {
 		_tag: "ok",
 		value: {
 			urls: result.entries.map((entry) => entry.loc),
-			complete: result.complete,
+			complete: result._tag === "complete",
 			documentsRead: result.documentsRead
 		}
 	};
 }
-export { loadSitemapUrls };
+async function discoverLiveSitemap(input, fetch = globalThis.fetch) {
+	let origin;
+	try {
+		origin = new URL(/^https?:\/\//i.test(input) ? input : `https://${input}`).origin;
+	} catch {
+		return {
+			_tag: "incomplete",
+			failures: [`${input}: invalid site URL`]
+		};
+	}
+	const reader = createLiveReader(fetch);
+	const failures = [];
+	for (const path of COMMON_PATHS) {
+		const result = await reader.read(`${origin}${path}`);
+		if (result._tag === "ok") return {
+			_tag: "found",
+			url: result.url,
+			source: "common_path"
+		};
+		if (result._tag !== "not_found" && !(result._tag === "failure" && result.reason === "document")) failures.push(readFailure(result));
+	}
+	const robotsUrl = `${origin}/robots.txt`;
+	const robots = await fetch(robotsUrl, { redirect: "manual" }).then(async (response) => response.ok ? response.text() : "").catch((error) => {
+		failures.push(`${robotsUrl}: ${error instanceof Error ? error.message : String(error)}`);
+		return "";
+	});
+	for (const reference of parseRobotsSitemaps(robots, robotsUrl)) {
+		const result = await reader.read(reference.loc);
+		if (result._tag === "ok") return {
+			_tag: "found",
+			url: result.url,
+			source: "robots"
+		};
+		if (result._tag !== "not_found") failures.push(readFailure(result));
+	}
+	return failures.length > 0 ? {
+		_tag: "incomplete",
+		failures
+	} : { _tag: "not_found" };
+}
+export { discoverLiveSitemap, loadSitemapUrls };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/cli",
   "type": "module",
-  "version": "1.5.4",
+  "version": "2.0.1",
   "description": "CLI for Google Search Console - dump, query, and run MCP server",
   "author": {
     "name": "Harlan Wilton",
@@ -49,11 +49,13 @@
     "google-auth-library": "^10.9.1",
     "ofetch": "^1.5.1",
     "open": "^11.0.0",
+    "sitemapd": "^0.1.0",
     "zod": "^4.4.3",
-    "@gscdump/analysis": "^1.5.4",
-    "@gscdump/engine": "^1.5.4",
-    "gscdump": "^1.5.4",
-    "@gscdump/engine-gsc-api": "^1.5.4"
+    "@gscdump/analysis": "^2.0.1",
+    "@gscdump/engine": "^2.0.1",
+    "@gscdump/engine-gsc-api": "^2.0.1",
+    "@gscdump/sdk": "^2.0.1",
+    "gscdump": "^2.0.1"
   },
   "devDependencies": {
     "vitest": "^4.1.10"