gscdump 0.8.0 → 0.8.2

package/dist/index.d.mts

@@ -1,6 +1,17 @@
 import { $Fetch, FetchOptions } from "ofetch";
 import { indexing_v3 } from "@googleapis/indexing/build/v3";
 import { searchconsole_v1 } from "@googleapis/searchconsole/build/v1";
+/**
+ * Batch runner with optional concurrency, inter-call delay, and progress.
+ * Used by batchRequestIndexing / batchInspectUrls. Defaults to sequential
+ * (concurrency = 1) because the underlying APIs rate-limit aggressively;
+ * callers that know their quota headroom can opt into parallelism.
+ */
+declare function runSequentialBatch<I, R>(items: I[], operation: (item: I, index: number) => Promise<R>, options?: {
+  delayMs?: number;
+  concurrency?: number;
+  onProgress?: (result: R, index: number, total: number) => void;
+}): Promise<R[]>;
 type ApiSite = searchconsole_v1.Schema$WmxSite;
 type ApiSitemap = searchconsole_v1.Schema$WmxSitemap;
 type ApiSitemapContent = searchconsole_v1.Schema$WmxSitemapContent;
@@ -162,6 +173,21 @@ interface GSCQueryBuilder<D extends Dimension[] = [], C = object> {
   toBody: () => SearchAnalyticsQuery;
   getState: () => BuilderState;
 }
+type VerificationMethod = 'META' | 'FILE' | 'DNS_TXT' | 'DNS_CNAME' | 'ANALYTICS' | 'TAG_MANAGER';
+type VerificationSiteType = 'SITE' | 'INET_DOMAIN' | 'ANDROID_APP';
+interface VerificationSite {
+  type: VerificationSiteType;
+  identifier: string;
+}
+interface VerificationToken {
+  method: string;
+  token: string;
+}
+interface VerificationWebResource {
+  id?: string;
+  site: VerificationSite;
+  owners?: string[];
+}
 /**
  * Compatible interface with OAuth2Client from google-auth-library
  */
@@ -193,8 +219,30 @@ interface CallOptions {
 interface GoogleSearchConsoleClient {
   /** Query search analytics with builder, returns async generator yielding typed row batches */
   query: <D extends Dimension[], C>(siteUrl: string, builder: GSCQueryBuilder<D, C>, opts?: CallOptions) => AsyncGenerator<GSCRow<D, C>[]>;
-  /** List all sites */
-  sites: (opts?: CallOptions) => Promise<ApiSite[]>;
+  /**
+   * List all sites. Also exposes write ops as `client.sites.add(siteUrl)` and
+   * `client.sites.delete(siteUrl)`. Calling `client.sites()` is equivalent to
+   * `client.sites.list()`.
+   */
+  sites: ((opts?: CallOptions) => Promise<ApiSite[]>) & {
+    list: (opts?: CallOptions) => Promise<ApiSite[]>; /** Add a property in unverified state. Caller must verify ownership separately. */
+    add: (siteUrl: string, opts?: CallOptions) => Promise<void>; /** Remove a property from the user's account. */
+    delete: (siteUrl: string, opts?: CallOptions) => Promise<void>;
+  };
+  /** Site Verification API (siteverification.googleapis.com). Required to flip a property from unverified to verified. */
+  verification: {
+    /** Returns the token to place on the site/DNS, plus the resolved method. */getToken: (params: {
+      site: VerificationSite;
+      verificationMethod: VerificationMethod;
+    }, opts?: CallOptions) => Promise<VerificationToken>; /** Triggers Google to fetch + validate; returns the verified WebResource. */
+    insert: (params: {
+      site: VerificationSite;
+      verificationMethod: VerificationMethod;
+    }, opts?: CallOptions) => Promise<VerificationWebResource>;
+    list: (opts?: CallOptions) => Promise<VerificationWebResource[]>;
+    get: (id: string, opts?: CallOptions) => Promise<VerificationWebResource>;
+    delete: (id: string, opts?: CallOptions) => Promise<void>;
+  };
   /** Inspect a URL */
   inspect: (siteUrl: string, url: string, opts?: CallOptions) => Promise<InspectUrlIndexResponse>;
   /** Sitemap operations */
@@ -251,6 +299,7 @@ declare function getIndexingMetadata(client: GoogleSearchConsoleClient, url: str
 declare function batchRequestIndexing(client: GoogleSearchConsoleClient, urls: string[], options?: {
   type?: IndexingNotificationType;
   delayMs?: number;
+  concurrency?: number;
   onProgress?: (result: IndexingResult, index: number, total: number) => void;
 }): Promise<IndexingResult[]>;
 interface InspectUrlResult {
@@ -271,6 +320,7 @@ declare function inspectUrl(client: GoogleSearchConsoleClient, siteUrl: string,
  */
 declare function batchInspectUrls(client: GoogleSearchConsoleClient, siteUrl: string, urls: string[], options?: {
   delayMs?: number;
+  concurrency?: number;
   onProgress?: (result: InspectUrlResult, index: number, total: number) => void;
 }): Promise<InspectUrlResult[]>;
 /**
@@ -299,6 +349,54 @@ declare function submitSitemap(client: GoogleSearchConsoleClient, siteUrl: strin
  * Deletes a sitemap from Google Search Console.
  */
 declare function deleteSitemap(client: GoogleSearchConsoleClient, siteUrl: string, feedpath: string): Promise<void>;
+/**
+ * Add a property to the user's Search Console account.
+ *
+ * Note: this only registers the property in an unverified state. Ownership
+ * must be proven via the Site Verification API (see `verifySite`) before any
+ * data is accessible.
+ */
+declare function addSite(client: GoogleSearchConsoleClient, siteUrl: string): Promise<void>;
+/**
+ * Remove a property from the user's Search Console account.
+ */
+declare function deleteSite(client: GoogleSearchConsoleClient, siteUrl: string): Promise<void>;
+/**
+ * Resolve a Search Console site URL (`https://example.com/` or
+ * `sc-domain:example.com`) to the Site Verification API's site shape.
+ */
+declare function siteUrlToVerificationSite(siteUrl: string): VerificationSite;
+/**
+ * Methods valid for a given site shape. SITE properties can use META/FILE/
+ * ANALYTICS/TAG_MANAGER; INET_DOMAIN must use DNS_TXT or DNS_CNAME.
+ */
+declare function verificationMethodsFor(site: VerificationSite): VerificationMethod[];
+/**
+ * Get the verification token Google expects to find on the site or DNS.
+ */
+declare function getVerificationToken(client: GoogleSearchConsoleClient, siteUrl: string, method: VerificationMethod): Promise<VerificationToken & {
+  site: VerificationSite;
+}>;
+/**
+ * Trigger Google to validate the placed token. Caller is responsible for
+ * having placed the token (HTML tag / file / DNS record) before calling.
+ */
+declare function verifySite(client: GoogleSearchConsoleClient, siteUrl: string, method: VerificationMethod): Promise<VerificationWebResource>;
+/**
+ * List all verified WebResources for the authed user.
+ */
+declare function listVerifiedSites(client: GoogleSearchConsoleClient): Promise<VerificationWebResource[]>;
+/**
+ * Fetch a single verified WebResource by id.
+ */
+declare function getVerifiedSite(client: GoogleSearchConsoleClient, id: string): Promise<VerificationWebResource>;
+/**
+ * Drop the calling user's verified ownership of a WebResource. The placed
+ * verification token (meta tag / file / DNS record) MUST be removed first,
+ * otherwise Google may auto-re-verify and the call will fail. Other owners
+ * on the property are unaffected.
+ */
+declare function unverifySite(client: GoogleSearchConsoleClient, id: string): Promise<void>;
 interface GscdumpApiOptions {
   /** API key (gsd_user_xxx or gsd_prod_xxx) */
   apiKey: string;
@@ -463,4 +561,29 @@ declare const INDEXING_EFFECTIVE_LIMIT = 1800;
 declare function hasGscReadScope(scopes: string | null | undefined): boolean;
 declare function hasGscWriteScope(scopes: string | null | undefined): boolean;
 declare function hasIndexingScope(scopes: string | null | undefined): boolean;
-export { ApiSite, ApiSitemap, ApiSitemapContent, Auth, AuthClient, AuthOptions, BackfillProgress, CallOptions, DAYS_PER_RANGE, DataRow, DimensionFilter, DimensionFilterGroup, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, GscError, GscErrorKind, GscdumpApiOptions, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, IndexStatusResult, IndexingIssueType, IndexingMetadata, IndexingNotificationType, IndexingResult, InspectUrlIndexResponse, InspectUrlResult, MS_PER_DAY, MobileUsabilityResult, Period, PublishUrlNotificationResponse, RequiredNonNullable, ResolvedAnalyticsRange, RichResultsResult, SearchAnalyticsQuery, SearchAnalyticsResponse, Site, SiteAnalytics, UrlInspectionResult, UrlNotificationMetadata, addDays, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, progressBar, requestIndexing, rowWithMetricDefaults, storageError, submitSitemap, toIsoDate };
+interface DiscoverSitemapOptions {
+  /** User-Agent sent on the discovery requests. */
+  userAgent?: string;
+  /** AbortSignal threaded through fetches; defaults to a 10s timeout per call. */
+  signal?: AbortSignal;
+}
+/**
+ * Try to discover a sitemap for `domain` by checking robots.txt for a
+ * `Sitemap:` directive, then a small set of common paths. Returns the first
+ * URL that responds with a 2xx, or `null`.
+ */
+declare function discoverSitemap(domain: string, options?: DiscoverSitemapOptions): Promise<string | null>;
+interface FetchSitemapUrlsOptions extends DiscoverSitemapOptions {
+  /** Maximum nested sitemap-index depth to follow. Default 3. */
+  maxDepth?: number;
+  /** Stop after this many URLs (across all nested sitemaps). Default unlimited. */
+  limit?: number;
+}
+/**
+ * Fetch a sitemap (or sitemap index) and return the list of `<loc>` URLs.
+ * Sitemap-index files are followed up to `maxDepth` levels. Duplicates are
+ * de-duplicated. The XML parser is regex-based — it handles the common
+ * `<loc>https://...</loc>` shape but doesn't validate the schema.
+ */
+declare function fetchSitemapUrls(sitemapUrl: string, options?: FetchSitemapUrlsOptions): Promise<string[]>;
+export { ApiSite, ApiSitemap, ApiSitemapContent, Auth, AuthClient, AuthOptions, BackfillProgress, CallOptions, DAYS_PER_RANGE, DataRow, DimensionFilter, DimensionFilterGroup, DiscoverSitemapOptions, FetchSitemapUrlsOptions, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, GscError, GscErrorKind, GscdumpApiOptions, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, IndexStatusResult, IndexingIssueType, IndexingMetadata, IndexingNotificationType, IndexingResult, InspectUrlIndexResponse, InspectUrlResult, MS_PER_DAY, MobileUsabilityResult, Period, PublishUrlNotificationResponse, RequiredNonNullable, ResolvedAnalyticsRange, RichResultsResult, SearchAnalyticsQuery, SearchAnalyticsResponse, Site, SiteAnalytics, UrlInspectionResult, UrlNotificationMetadata, VerificationMethod, VerificationSite, VerificationSiteType, VerificationToken, VerificationWebResource, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, discoverSitemap, fetchSitemap, fetchSitemapUrls, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, getVerifiedSite, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, runSequentialBatch, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, unverifySite, verificationMethodsFor, verifySite };

package/dist/index.mjs

@@ -1,13 +1,30 @@
 import { ofetch } from "ofetch";
 async function runSequentialBatch(items, operation, options = {}) {
-	const { delayMs = 0, onProgress } = options;
-	const results = [];
-	for (let i = 0; i < items.length; i++) {
-		const result = await operation(items[i], i);
-		results.push(result);
-		onProgress?.(result, i, items.length);
-		if (i < items.length - 1 && delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+	const { delayMs = 0, concurrency = 1, onProgress } = options;
+	const results = Array.from({ length: items.length });
+	let completed = 0;
+	if (concurrency <= 1) {
+		for (let i = 0; i < items.length; i++) {
+			const result = await operation(items[i], i);
+			results[i] = result;
+			onProgress?.(result, i, items.length);
+			if (i < items.length - 1 && delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+		}
+		return results;
 	}
+	const cursor = { i: 0 };
+	const worker = async () => {
+		while (true) {
+			const i = cursor.i++;
+			if (i >= items.length) return;
+			const result = await operation(items[i], i);
+			results[i] = result;
+			completed++;
+			onProgress?.(result, completed - 1, items.length);
+			if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
 	return results;
 }
 async function requestIndexing(client, url, options = {}) {
@@ -26,9 +43,10 @@ async function getIndexingMetadata(client, url) {
 	}));
 }
 async function batchRequestIndexing(client, urls, options = {}) {
-	const { type = "URL_UPDATED", delayMs = 100, onProgress } = options;
+	const { type = "URL_UPDATED", delayMs = 100, concurrency, onProgress } = options;
 	return runSequentialBatch(urls, (url) => requestIndexing(client, url, { type }), {
 		delayMs,
+		concurrency,
 		onProgress
 	});
 }
@@ -40,7 +58,7 @@ async function inspectUrl(client, siteUrl, inspectionUrl) {
 	};
 }
 async function batchInspectUrls(client, siteUrl, urls, options = {}) {
-	const { delayMs = 200, onProgress } = options;
+	const { delayMs = 200, concurrency, onProgress } = options;
 	return runSequentialBatch(urls, async (url) => {
 		const { inspection, isIndexed } = await inspectUrl(client, siteUrl, url);
 		return {
@@ -50,6 +68,7 @@ async function batchInspectUrls(client, siteUrl, urls, options = {}) {
 		};
 	}, {
 		delayMs,
+		concurrency,
 		onProgress
 	});
 }
@@ -78,6 +97,58 @@ async function submitSitemap(client, siteUrl, feedpath) {
 async function deleteSitemap(client, siteUrl, feedpath) {
 	return client.sitemaps.delete(siteUrl, feedpath);
 }
+async function addSite(client, siteUrl) {
+	return client.sites.add(siteUrl);
+}
+async function deleteSite(client, siteUrl) {
+	return client.sites.delete(siteUrl);
+}
+const SC_DOMAIN_PREFIX = "sc-domain:";
+function siteUrlToVerificationSite(siteUrl) {
+	if (siteUrl.startsWith(SC_DOMAIN_PREFIX)) return {
+		type: "INET_DOMAIN",
+		identifier: siteUrl.slice(10)
+	};
+	return {
+		type: "SITE",
+		identifier: siteUrl
+	};
+}
+function verificationMethodsFor(site) {
+	if (site.type === "INET_DOMAIN") return ["DNS_TXT", "DNS_CNAME"];
+	return [
+		"META",
+		"FILE",
+		"ANALYTICS",
+		"TAG_MANAGER"
+	];
+}
+async function getVerificationToken(client, siteUrl, method) {
+	const site = siteUrlToVerificationSite(siteUrl);
+	return {
+		...await client.verification.getToken({
+			site,
+			verificationMethod: method
+		}),
+		site
+	};
+}
+async function verifySite(client, siteUrl, method) {
+	const site = siteUrlToVerificationSite(siteUrl);
+	return client.verification.insert({
+		site,
+		verificationMethod: method
+	});
+}
+async function listVerifiedSites(client) {
+	return client.verification.list();
+}
+async function getVerifiedSite(client, id) {
+	return client.verification.get(id);
+}
+async function unverifySite(client, id) {
+	return client.verification.delete(id);
+}
 const MS_PER_DAY = 864e5;
 function toIsoDate(d) {
 	return d.toISOString().slice(0, 10);
@@ -377,11 +448,38 @@ function gscdumpApi(options) {
 				startRow += rows.length;
 			}
 		},
-		sites: async (opts) => {
-			return (await fetch("/api/sites", { signal: opts?.signal })).sites.map((s) => ({
-				siteUrl: s.gscSiteUrl,
-				permissionLevel: s.permissionLevel || "siteOwner"
-			}));
+		sites: (() => {
+			const list = async (opts) => {
+				return (await fetch("/api/sites", { signal: opts?.signal })).sites.map((s) => ({
+					siteUrl: s.gscSiteUrl,
+					permissionLevel: s.permissionLevel || "siteOwner"
+				}));
+			};
+			const unsupported = (op) => () => {
+				throw new Error(`sites.${op} not available via gscdump API. Use googleSearchConsole() with OAuth credentials.`);
+			};
+			return Object.assign(list, {
+				list,
+				add: unsupported("add"),
+				delete: unsupported("delete")
+			});
+		})(),
+		verification: {
+			getToken: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			insert: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			list: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			get: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			delete: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			}
 		},
 		inspect: () => {
 			throw new Error("URL inspection not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
@@ -413,6 +511,7 @@ function gscdumpApi(options) {
 }
 const GSC_API = "https://searchconsole.googleapis.com";
 const INDEXING_API = "https://indexing.googleapis.com";
+const SITE_VERIFICATION_API = "https://www.googleapis.com/siteVerification/v1";
 function createAuth(options) {
 	let credentials = { refresh_token: options.refreshToken };
 	return {
@@ -529,8 +628,42 @@ function googleSearchConsole(auth, options = {}) {
 				startRow += rows.length;
 			}
 		},
-		sites: async (opts) => {
-			return (await fetch(`${GSC_API}/webmasters/v3/sites`, { signal: opts?.signal })).siteEntry || [];
+		sites: (() => {
+			const list = async (opts) => {
+				return (await fetch(`${GSC_API}/webmasters/v3/sites`, { signal: opts?.signal })).siteEntry || [];
+			};
+			return Object.assign(list, {
+				list,
+				add: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeURIComponent(siteUrl)}`, {
+					method: "PUT",
+					signal: opts?.signal
+				}),
+				delete: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeURIComponent(siteUrl)}`, {
+					method: "DELETE",
+					signal: opts?.signal
+				})
+			});
+		})(),
+		verification: {
+			getToken: (params, opts) => fetch(`${SITE_VERIFICATION_API}/token`, {
+				method: "POST",
+				body: params,
+				signal: opts?.signal
+			}),
+			insert: (params, opts) => fetch(`${SITE_VERIFICATION_API}/webResource`, {
+				method: "POST",
+				query: { verificationMethod: params.verificationMethod },
+				body: { site: params.site },
+				signal: opts?.signal
+			}),
+			list: async (opts) => {
+				return (await fetch(`${SITE_VERIFICATION_API}/webResource`, { signal: opts?.signal })).items || [];
+			},
+			get: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, { signal: opts?.signal }),
+			delete: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
+				method: "DELETE",
+				signal: opts?.signal
+			})
 		},
 		inspect: (siteUrl, url, opts) => fetch(`${GSC_API}/v1/urlInspection/index:inspect`, {
 			method: "POST",
@@ -796,4 +929,71 @@ function hasIndexingScope(scopes) {
 	if (!scopes) return false;
 	return scopes.includes("googleapis.com/auth/indexing");
 }
-export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, MS_PER_DAY, addDays, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, progressBar, requestIndexing, rowWithMetricDefaults, storageError, submitSitemap, toIsoDate };
+const FETCH_TIMEOUT_MS = 1e4;
+const COMMON_PATHS = ["/sitemap.xml", "/sitemap_index.xml"];
+const SITEMAP_DIRECTIVE_RE = /^Sitemap:\s*(\S+)/im;
+async function discoverSitemap(domain, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const baseUrl = `https://${domain}`;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const robotsRes = await fetch(`${baseUrl}/robots.txt`, {
+		headers: { "User-Agent": userAgent },
+		signal: signalFor()
+	}).catch(() => null);
+	if (robotsRes?.ok) {
+		const match = (await robotsRes.text()).match(SITEMAP_DIRECTIVE_RE);
+		if (match?.[1]) {
+			if ((await fetch(match[1], {
+				method: "HEAD",
+				signal: signalFor()
+			}).catch(() => null))?.ok) return match[1];
+		}
+	}
+	for (const path of COMMON_PATHS) {
+		const url = `${baseUrl}${path}`;
+		if ((await fetch(url, {
+			method: "HEAD",
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		}).catch(() => null))?.ok) return url;
+	}
+	return null;
+}
+const LOC_RE = /<loc>([^<]+)<\/loc>/gi;
+const SITEMAPINDEX_RE = /<sitemapindex\b/i;
+async function fetchSitemapUrls(sitemapUrl, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const maxDepth = options.maxDepth ?? 3;
+	const limit = options.limit;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	const visit = async (url, depth) => {
+		if (limit != null && out.length >= limit) return;
+		if (depth > maxDepth) return;
+		const res = await fetch(url, {
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		});
+		if (!res.ok) throw new Error(`Fetch ${url} failed: ${res.status}`);
+		const text = await res.text();
+		const isIndex = SITEMAPINDEX_RE.test(text);
+		const matches = [...text.matchAll(LOC_RE)].map((m) => m[1].trim()).filter(Boolean);
+		if (isIndex) {
+			for (const child of matches) {
+				if (limit != null && out.length >= limit) return;
+				await visit(child, depth + 1);
+			}
+			return;
+		}
+		for (const u of matches) {
+			if (seen.has(u)) continue;
+			seen.add(u);
+			out.push(u);
+			if (limit != null && out.length >= limit) return;
+		}
+	};
+	await visit(sitemapUrl, 0);
+	return out;
+}
+export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, MS_PER_DAY, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, discoverSitemap, fetchSitemap, fetchSitemapUrls, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, getVerifiedSite, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, runSequentialBatch, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, unverifySite, verificationMethodsFor, verifySite };

package/dist/sitemap.d.mts

@@ -10,4 +10,17 @@ interface DiscoverSitemapOptions {
  * URL that responds with a 2xx, or `null`.
  */
 declare function discoverSitemap(domain: string, options?: DiscoverSitemapOptions): Promise<string | null>;
-export { DiscoverSitemapOptions, discoverSitemap };
+interface FetchSitemapUrlsOptions extends DiscoverSitemapOptions {
+  /** Maximum nested sitemap-index depth to follow. Default 3. */
+  maxDepth?: number;
+  /** Stop after this many URLs (across all nested sitemaps). Default unlimited. */
+  limit?: number;
+}
+/**
+ * Fetch a sitemap (or sitemap index) and return the list of `<loc>` URLs.
+ * Sitemap-index files are followed up to `maxDepth` levels. Duplicates are
+ * de-duplicated. The XML parser is regex-based — it handles the common
+ * `<loc>https://...</loc>` shape but doesn't validate the schema.
+ */
+declare function fetchSitemapUrls(sitemapUrl: string, options?: FetchSitemapUrlsOptions): Promise<string[]>;
+export { DiscoverSitemapOptions, FetchSitemapUrlsOptions, discoverSitemap, fetchSitemapUrls };

package/dist/sitemap.mjs

@@ -28,4 +28,41 @@ async function discoverSitemap(domain, options = {}) {
 	}
 	return null;
 }
-export { discoverSitemap };
+const LOC_RE = /<loc>([^<]+)<\/loc>/gi;
+const SITEMAPINDEX_RE = /<sitemapindex\b/i;
+async function fetchSitemapUrls(sitemapUrl, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const maxDepth = options.maxDepth ?? 3;
+	const limit = options.limit;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	const visit = async (url, depth) => {
+		if (limit != null && out.length >= limit) return;
+		if (depth > maxDepth) return;
+		const res = await fetch(url, {
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		});
+		if (!res.ok) throw new Error(`Fetch ${url} failed: ${res.status}`);
+		const text = await res.text();
+		const isIndex = SITEMAPINDEX_RE.test(text);
+		const matches = [...text.matchAll(LOC_RE)].map((m) => m[1].trim()).filter(Boolean);
+		if (isIndex) {
+			for (const child of matches) {
+				if (limit != null && out.length >= limit) return;
+				await visit(child, depth + 1);
+			}
+			return;
+		}
+		for (const u of matches) {
+			if (seen.has(u)) continue;
+			seen.add(u);
+			out.push(u);
+			if (limit != null && out.length >= limit) return;
+		}
+	};
+	await visit(sitemapUrl, 0);
+	return out;
+}
+export { discoverSitemap, fetchSitemapUrls };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gscdump",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Google Search Console API wrapper with typed query builder, streaming pagination, and SEO analysis functions",
   "author": {
     "name": "Harlan Wilton",
@@ -102,7 +102,7 @@
     "dayjs": "^1.11.20",
     "defu": "^6.1.7",
     "ofetch": "^1.5.1",
-    "ufo": "^1.6.3"
+    "ufo": "^1.6.4"
   },
   "devDependencies": {
     "@googleapis/indexing": "^6.0.1",