gscdump 0.8.1 → 0.9.0

package/dist/index.d.mts

@@ -1,6 +1,17 @@
 import { $Fetch, FetchOptions } from "ofetch";
 import { indexing_v3 } from "@googleapis/indexing/build/v3";
 import { searchconsole_v1 } from "@googleapis/searchconsole/build/v1";
+/**
+ * Batch runner with optional concurrency, inter-call delay, and progress.
+ * Used by batchRequestIndexing / batchInspectUrls. Defaults to sequential
+ * (concurrency = 1) because the underlying APIs rate-limit aggressively;
+ * callers that know their quota headroom can opt into parallelism.
+ */
+declare function runSequentialBatch<I, R>(items: I[], operation: (item: I, index: number) => Promise<R>, options?: {
+  delayMs?: number;
+  concurrency?: number;
+  onProgress?: (result: R, index: number, total: number) => void;
+}): Promise<R[]>;
 type ApiSite = searchconsole_v1.Schema$WmxSite;
 type ApiSitemap = searchconsole_v1.Schema$WmxSitemap;
 type ApiSitemapContent = searchconsole_v1.Schema$WmxSitemapContent;
@@ -375,6 +386,17 @@ declare function verifySite(client: GoogleSearchConsoleClient, siteUrl: string,
  * List all verified WebResources for the authed user.
  */
 declare function listVerifiedSites(client: GoogleSearchConsoleClient): Promise<VerificationWebResource[]>;
+/**
+ * Fetch a single verified WebResource by id.
+ */
+declare function getVerifiedSite(client: GoogleSearchConsoleClient, id: string): Promise<VerificationWebResource>;
+/**
+ * Drop the calling user's verified ownership of a WebResource. The placed
+ * verification token (meta tag / file / DNS record) MUST be removed first,
+ * otherwise Google may auto-re-verify and the call will fail. Other owners
+ * on the property are unaffected.
+ */
+declare function unverifySite(client: GoogleSearchConsoleClient, id: string): Promise<void>;
 interface GscdumpApiOptions {
   /** API key (gsd_user_xxx or gsd_prod_xxx) */
   apiKey: string;
@@ -539,4 +561,29 @@ declare const INDEXING_EFFECTIVE_LIMIT = 1800;
 declare function hasGscReadScope(scopes: string | null | undefined): boolean;
 declare function hasGscWriteScope(scopes: string | null | undefined): boolean;
 declare function hasIndexingScope(scopes: string | null | undefined): boolean;
-export { ApiSite, ApiSitemap, ApiSitemapContent, Auth, AuthClient, AuthOptions, BackfillProgress, CallOptions, DAYS_PER_RANGE, DataRow, DimensionFilter, DimensionFilterGroup, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, GscError, GscErrorKind, GscdumpApiOptions, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, IndexStatusResult, IndexingIssueType, IndexingMetadata, IndexingNotificationType, IndexingResult, InspectUrlIndexResponse, InspectUrlResult, MS_PER_DAY, MobileUsabilityResult, Period, PublishUrlNotificationResponse, RequiredNonNullable, ResolvedAnalyticsRange, RichResultsResult, SearchAnalyticsQuery, SearchAnalyticsResponse, Site, SiteAnalytics, UrlInspectionResult, UrlNotificationMetadata, VerificationMethod, VerificationSite, VerificationSiteType, VerificationToken, VerificationWebResource, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, verificationMethodsFor, verifySite };
+interface DiscoverSitemapOptions {
+  /** User-Agent sent on the discovery requests. */
+  userAgent?: string;
+  /** AbortSignal threaded through fetches; defaults to a 10s timeout per call. */
+  signal?: AbortSignal;
+}
+/**
+ * Try to discover a sitemap for `domain` by checking robots.txt for a
+ * `Sitemap:` directive, then a small set of common paths. Returns the first
+ * URL that responds with a 2xx, or `null`.
+ */
+declare function discoverSitemap(domain: string, options?: DiscoverSitemapOptions): Promise<string | null>;
+interface FetchSitemapUrlsOptions extends DiscoverSitemapOptions {
+  /** Maximum nested sitemap-index depth to follow. Default 3. */
+  maxDepth?: number;
+  /** Stop after this many URLs (across all nested sitemaps). Default unlimited. */
+  limit?: number;
+}
+/**
+ * Fetch a sitemap (or sitemap index) and return the list of `<loc>` URLs.
+ * Sitemap-index files are followed up to `maxDepth` levels. Duplicates are
+ * de-duplicated. The XML parser is regex-based — it handles the common
+ * `<loc>https://...</loc>` shape but doesn't validate the schema.
+ */
+declare function fetchSitemapUrls(sitemapUrl: string, options?: FetchSitemapUrlsOptions): Promise<string[]>;
+export { ApiSite, ApiSitemap, ApiSitemapContent, Auth, AuthClient, AuthOptions, BackfillProgress, CallOptions, DAYS_PER_RANGE, DataRow, DimensionFilter, DimensionFilterGroup, DiscoverSitemapOptions, FetchSitemapUrlsOptions, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, GscError, GscErrorKind, GscdumpApiOptions, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, IndexStatusResult, IndexingIssueType, IndexingMetadata, IndexingNotificationType, IndexingResult, InspectUrlIndexResponse, InspectUrlResult, MS_PER_DAY, MobileUsabilityResult, Period, PublishUrlNotificationResponse, RequiredNonNullable, ResolvedAnalyticsRange, RichResultsResult, SearchAnalyticsQuery, SearchAnalyticsResponse, Site, SiteAnalytics, UrlInspectionResult, UrlNotificationMetadata, VerificationMethod, VerificationSite, VerificationSiteType, VerificationToken, VerificationWebResource, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, discoverSitemap, fetchSitemap, fetchSitemapUrls, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, getVerifiedSite, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, runSequentialBatch, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, unverifySite, verificationMethodsFor, verifySite };

package/dist/index.mjs

@@ -143,6 +143,12 @@ async function verifySite(client, siteUrl, method) {
 async function listVerifiedSites(client) {
 	return client.verification.list();
 }
+async function getVerifiedSite(client, id) {
+	return client.verification.get(id);
+}
+async function unverifySite(client, id) {
+	return client.verification.delete(id);
+}
 const MS_PER_DAY = 864e5;
 function toIsoDate(d) {
 	return d.toISOString().slice(0, 10);
@@ -923,4 +929,71 @@ function hasIndexingScope(scopes) {
 	if (!scopes) return false;
 	return scopes.includes("googleapis.com/auth/indexing");
 }
-export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, MS_PER_DAY, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, verificationMethodsFor, verifySite };
+const FETCH_TIMEOUT_MS = 1e4;
+const COMMON_PATHS = ["/sitemap.xml", "/sitemap_index.xml"];
+const SITEMAP_DIRECTIVE_RE = /^Sitemap:\s*(\S+)/im;
+async function discoverSitemap(domain, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const baseUrl = `https://${domain}`;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const robotsRes = await fetch(`${baseUrl}/robots.txt`, {
+		headers: { "User-Agent": userAgent },
+		signal: signalFor()
+	}).catch(() => null);
+	if (robotsRes?.ok) {
+		const match = (await robotsRes.text()).match(SITEMAP_DIRECTIVE_RE);
+		if (match?.[1]) {
+			if ((await fetch(match[1], {
+				method: "HEAD",
+				signal: signalFor()
+			}).catch(() => null))?.ok) return match[1];
+		}
+	}
+	for (const path of COMMON_PATHS) {
+		const url = `${baseUrl}${path}`;
+		if ((await fetch(url, {
+			method: "HEAD",
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		}).catch(() => null))?.ok) return url;
+	}
+	return null;
+}
+const LOC_RE = /<loc>([^<]+)<\/loc>/gi;
+const SITEMAPINDEX_RE = /<sitemapindex\b/i;
+async function fetchSitemapUrls(sitemapUrl, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const maxDepth = options.maxDepth ?? 3;
+	const limit = options.limit;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	const visit = async (url, depth) => {
+		if (limit != null && out.length >= limit) return;
+		if (depth > maxDepth) return;
+		const res = await fetch(url, {
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		});
+		if (!res.ok) throw new Error(`Fetch ${url} failed: ${res.status}`);
+		const text = await res.text();
+		const isIndex = SITEMAPINDEX_RE.test(text);
+		const matches = [...text.matchAll(LOC_RE)].map((m) => m[1].trim()).filter(Boolean);
+		if (isIndex) {
+			for (const child of matches) {
+				if (limit != null && out.length >= limit) return;
+				await visit(child, depth + 1);
+			}
+			return;
+		}
+		for (const u of matches) {
+			if (seen.has(u)) continue;
+			seen.add(u);
+			out.push(u);
+			if (limit != null && out.length >= limit) return;
+		}
+	};
+	await visit(sitemapUrl, 0);
+	return out;
+}
+export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, MS_PER_DAY, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, discoverSitemap, fetchSitemap, fetchSitemapUrls, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, getVerifiedSite, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, runSequentialBatch, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, unverifySite, verificationMethodsFor, verifySite };

package/dist/sitemap.d.mts

@@ -10,4 +10,17 @@ interface DiscoverSitemapOptions {
  * URL that responds with a 2xx, or `null`.
  */
 declare function discoverSitemap(domain: string, options?: DiscoverSitemapOptions): Promise<string | null>;
-export { DiscoverSitemapOptions, discoverSitemap };
+interface FetchSitemapUrlsOptions extends DiscoverSitemapOptions {
+  /** Maximum nested sitemap-index depth to follow. Default 3. */
+  maxDepth?: number;
+  /** Stop after this many URLs (across all nested sitemaps). Default unlimited. */
+  limit?: number;
+}
+/**
+ * Fetch a sitemap (or sitemap index) and return the list of `<loc>` URLs.
+ * Sitemap-index files are followed up to `maxDepth` levels. Duplicates are
+ * de-duplicated. The XML parser is regex-based — it handles the common
+ * `<loc>https://...</loc>` shape but doesn't validate the schema.
+ */
+declare function fetchSitemapUrls(sitemapUrl: string, options?: FetchSitemapUrlsOptions): Promise<string[]>;
+export { DiscoverSitemapOptions, FetchSitemapUrlsOptions, discoverSitemap, fetchSitemapUrls };

package/dist/sitemap.mjs

@@ -28,4 +28,41 @@ async function discoverSitemap(domain, options = {}) {
 	}
 	return null;
 }
-export { discoverSitemap };
+const LOC_RE = /<loc>([^<]+)<\/loc>/gi;
+const SITEMAPINDEX_RE = /<sitemapindex\b/i;
+async function fetchSitemapUrls(sitemapUrl, options = {}) {
+	const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
+	const maxDepth = options.maxDepth ?? 3;
+	const limit = options.limit;
+	const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	const visit = async (url, depth) => {
+		if (limit != null && out.length >= limit) return;
+		if (depth > maxDepth) return;
+		const res = await fetch(url, {
+			headers: { "User-Agent": userAgent },
+			signal: signalFor()
+		});
+		if (!res.ok) throw new Error(`Fetch ${url} failed: ${res.status}`);
+		const text = await res.text();
+		const isIndex = SITEMAPINDEX_RE.test(text);
+		const matches = [...text.matchAll(LOC_RE)].map((m) => m[1].trim()).filter(Boolean);
+		if (isIndex) {
+			for (const child of matches) {
+				if (limit != null && out.length >= limit) return;
+				await visit(child, depth + 1);
+			}
+			return;
+		}
+		for (const u of matches) {
+			if (seen.has(u)) continue;
+			seen.add(u);
+			out.push(u);
+			if (limit != null && out.length >= limit) return;
+		}
+	};
+	await visit(sitemapUrl, 0);
+	return out;
+}
+export { discoverSitemap, fetchSitemapUrls };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gscdump",
   "type": "module",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "Google Search Console API wrapper with typed query builder, streaming pagination, and SEO analysis functions",
   "author": {
     "name": "Harlan Wilton",