gscdump 0.8.0 → 0.8.1

package/dist/index.d.mts

@@ -162,6 +162,21 @@ interface GSCQueryBuilder<D extends Dimension[] = [], C = object> {
   toBody: () => SearchAnalyticsQuery;
   getState: () => BuilderState;
 }
+type VerificationMethod = 'META' | 'FILE' | 'DNS_TXT' | 'DNS_CNAME' | 'ANALYTICS' | 'TAG_MANAGER';
+type VerificationSiteType = 'SITE' | 'INET_DOMAIN' | 'ANDROID_APP';
+interface VerificationSite {
+  type: VerificationSiteType;
+  identifier: string;
+}
+interface VerificationToken {
+  method: string;
+  token: string;
+}
+interface VerificationWebResource {
+  id?: string;
+  site: VerificationSite;
+  owners?: string[];
+}
 /**
  * Compatible interface with OAuth2Client from google-auth-library
  */
@@ -193,8 +208,30 @@ interface CallOptions {
 interface GoogleSearchConsoleClient {
   /** Query search analytics with builder, returns async generator yielding typed row batches */
   query: <D extends Dimension[], C>(siteUrl: string, builder: GSCQueryBuilder<D, C>, opts?: CallOptions) => AsyncGenerator<GSCRow<D, C>[]>;
-  /** List all sites */
-  sites: (opts?: CallOptions) => Promise<ApiSite[]>;
+  /**
+   * List all sites. Also exposes write ops as `client.sites.add(siteUrl)` and
+   * `client.sites.delete(siteUrl)`. Calling `client.sites()` is equivalent to
+   * `client.sites.list()`.
+   */
+  sites: ((opts?: CallOptions) => Promise<ApiSite[]>) & {
+    list: (opts?: CallOptions) => Promise<ApiSite[]>; /** Add a property in unverified state. Caller must verify ownership separately. */
+    add: (siteUrl: string, opts?: CallOptions) => Promise<void>; /** Remove a property from the user's account. */
+    delete: (siteUrl: string, opts?: CallOptions) => Promise<void>;
+  };
+  /** Site Verification API (siteverification.googleapis.com). Required to flip a property from unverified to verified. */
+  verification: {
+    /** Returns the token to place on the site/DNS, plus the resolved method. */getToken: (params: {
+      site: VerificationSite;
+      verificationMethod: VerificationMethod;
+    }, opts?: CallOptions) => Promise<VerificationToken>; /** Triggers Google to fetch + validate; returns the verified WebResource. */
+    insert: (params: {
+      site: VerificationSite;
+      verificationMethod: VerificationMethod;
+    }, opts?: CallOptions) => Promise<VerificationWebResource>;
+    list: (opts?: CallOptions) => Promise<VerificationWebResource[]>;
+    get: (id: string, opts?: CallOptions) => Promise<VerificationWebResource>;
+    delete: (id: string, opts?: CallOptions) => Promise<void>;
+  };
   /** Inspect a URL */
   inspect: (siteUrl: string, url: string, opts?: CallOptions) => Promise<InspectUrlIndexResponse>;
   /** Sitemap operations */
@@ -251,6 +288,7 @@ declare function getIndexingMetadata(client: GoogleSearchConsoleClient, url: str
 declare function batchRequestIndexing(client: GoogleSearchConsoleClient, urls: string[], options?: {
   type?: IndexingNotificationType;
   delayMs?: number;
+  concurrency?: number;
   onProgress?: (result: IndexingResult, index: number, total: number) => void;
 }): Promise<IndexingResult[]>;
 interface InspectUrlResult {
@@ -271,6 +309,7 @@ declare function inspectUrl(client: GoogleSearchConsoleClient, siteUrl: string,
  */
 declare function batchInspectUrls(client: GoogleSearchConsoleClient, siteUrl: string, urls: string[], options?: {
   delayMs?: number;
+  concurrency?: number;
   onProgress?: (result: InspectUrlResult, index: number, total: number) => void;
 }): Promise<InspectUrlResult[]>;
 /**
@@ -299,6 +338,43 @@ declare function submitSitemap(client: GoogleSearchConsoleClient, siteUrl: strin
  * Deletes a sitemap from Google Search Console.
  */
 declare function deleteSitemap(client: GoogleSearchConsoleClient, siteUrl: string, feedpath: string): Promise<void>;
+/**
+ * Add a property to the user's Search Console account.
+ *
+ * Note: this only registers the property in an unverified state. Ownership
+ * must be proven via the Site Verification API (see `verifySite`) before any
+ * data is accessible.
+ */
+declare function addSite(client: GoogleSearchConsoleClient, siteUrl: string): Promise<void>;
+/**
+ * Remove a property from the user's Search Console account.
+ */
+declare function deleteSite(client: GoogleSearchConsoleClient, siteUrl: string): Promise<void>;
+/**
+ * Resolve a Search Console site URL (`https://example.com/` or
+ * `sc-domain:example.com`) to the Site Verification API's site shape.
+ */
+declare function siteUrlToVerificationSite(siteUrl: string): VerificationSite;
+/**
+ * Methods valid for a given site shape. SITE properties can use META/FILE/
+ * ANALYTICS/TAG_MANAGER; INET_DOMAIN must use DNS_TXT or DNS_CNAME.
+ */
+declare function verificationMethodsFor(site: VerificationSite): VerificationMethod[];
+/**
+ * Get the verification token Google expects to find on the site or DNS.
+ */
+declare function getVerificationToken(client: GoogleSearchConsoleClient, siteUrl: string, method: VerificationMethod): Promise<VerificationToken & {
+  site: VerificationSite;
+}>;
+/**
+ * Trigger Google to validate the placed token. Caller is responsible for
+ * having placed the token (HTML tag / file / DNS record) before calling.
+ */
+declare function verifySite(client: GoogleSearchConsoleClient, siteUrl: string, method: VerificationMethod): Promise<VerificationWebResource>;
+/**
+ * List all verified WebResources for the authed user.
+ */
+declare function listVerifiedSites(client: GoogleSearchConsoleClient): Promise<VerificationWebResource[]>;
 interface GscdumpApiOptions {
   /** API key (gsd_user_xxx or gsd_prod_xxx) */
   apiKey: string;
@@ -463,4 +539,4 @@ declare const INDEXING_EFFECTIVE_LIMIT = 1800;
 declare function hasGscReadScope(scopes: string | null | undefined): boolean;
 declare function hasGscWriteScope(scopes: string | null | undefined): boolean;
 declare function hasIndexingScope(scopes: string | null | undefined): boolean;
-export { ApiSite, ApiSitemap, ApiSitemapContent, Auth, AuthClient, AuthOptions, BackfillProgress, CallOptions, DAYS_PER_RANGE, DataRow, DimensionFilter, DimensionFilterGroup, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, GscError, GscErrorKind, GscdumpApiOptions, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, IndexStatusResult, IndexingIssueType, IndexingMetadata, IndexingNotificationType, IndexingResult, InspectUrlIndexResponse, InspectUrlResult, MS_PER_DAY, MobileUsabilityResult, Period, PublishUrlNotificationResponse, RequiredNonNullable, ResolvedAnalyticsRange, RichResultsResult, SearchAnalyticsQuery, SearchAnalyticsResponse, Site, SiteAnalytics, UrlInspectionResult, UrlNotificationMetadata, addDays, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, progressBar, requestIndexing, rowWithMetricDefaults, storageError, submitSitemap, toIsoDate };
+export { ApiSite, ApiSitemap, ApiSitemapContent, Auth, AuthClient, AuthOptions, BackfillProgress, CallOptions, DAYS_PER_RANGE, DataRow, DimensionFilter, DimensionFilterGroup, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, GscError, GscErrorKind, GscdumpApiOptions, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, IndexStatusResult, IndexingIssueType, IndexingMetadata, IndexingNotificationType, IndexingResult, InspectUrlIndexResponse, InspectUrlResult, MS_PER_DAY, MobileUsabilityResult, Period, PublishUrlNotificationResponse, RequiredNonNullable, ResolvedAnalyticsRange, RichResultsResult, SearchAnalyticsQuery, SearchAnalyticsResponse, Site, SiteAnalytics, UrlInspectionResult, UrlNotificationMetadata, VerificationMethod, VerificationSite, VerificationSiteType, VerificationToken, VerificationWebResource, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, verificationMethodsFor, verifySite };

package/dist/index.mjs

@@ -1,13 +1,30 @@
 import { ofetch } from "ofetch";
 async function runSequentialBatch(items, operation, options = {}) {
-	const { delayMs = 0, onProgress } = options;
-	const results = [];
-	for (let i = 0; i < items.length; i++) {
-		const result = await operation(items[i], i);
-		results.push(result);
-		onProgress?.(result, i, items.length);
-		if (i < items.length - 1 && delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+	const { delayMs = 0, concurrency = 1, onProgress } = options;
+	const results = Array.from({ length: items.length });
+	let completed = 0;
+	if (concurrency <= 1) {
+		for (let i = 0; i < items.length; i++) {
+			const result = await operation(items[i], i);
+			results[i] = result;
+			onProgress?.(result, i, items.length);
+			if (i < items.length - 1 && delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+		}
+		return results;
 	}
+	const cursor = { i: 0 };
+	const worker = async () => {
+		while (true) {
+			const i = cursor.i++;
+			if (i >= items.length) return;
+			const result = await operation(items[i], i);
+			results[i] = result;
+			completed++;
+			onProgress?.(result, completed - 1, items.length);
+			if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
 	return results;
 }
 async function requestIndexing(client, url, options = {}) {
@@ -26,9 +43,10 @@ async function getIndexingMetadata(client, url) {
 	}));
 }
 async function batchRequestIndexing(client, urls, options = {}) {
-	const { type = "URL_UPDATED", delayMs = 100, onProgress } = options;
+	const { type = "URL_UPDATED", delayMs = 100, concurrency, onProgress } = options;
 	return runSequentialBatch(urls, (url) => requestIndexing(client, url, { type }), {
 		delayMs,
+		concurrency,
 		onProgress
 	});
 }
@@ -40,7 +58,7 @@ async function inspectUrl(client, siteUrl, inspectionUrl) {
 	};
 }
 async function batchInspectUrls(client, siteUrl, urls, options = {}) {
-	const { delayMs = 200, onProgress } = options;
+	const { delayMs = 200, concurrency, onProgress } = options;
 	return runSequentialBatch(urls, async (url) => {
 		const { inspection, isIndexed } = await inspectUrl(client, siteUrl, url);
 		return {
@@ -50,6 +68,7 @@ async function batchInspectUrls(client, siteUrl, urls, options = {}) {
 		};
 	}, {
 		delayMs,
+		concurrency,
 		onProgress
 	});
 }
@@ -78,6 +97,52 @@ async function submitSitemap(client, siteUrl, feedpath) {
 async function deleteSitemap(client, siteUrl, feedpath) {
 	return client.sitemaps.delete(siteUrl, feedpath);
 }
+async function addSite(client, siteUrl) {
+	return client.sites.add(siteUrl);
+}
+async function deleteSite(client, siteUrl) {
+	return client.sites.delete(siteUrl);
+}
+const SC_DOMAIN_PREFIX = "sc-domain:";
+function siteUrlToVerificationSite(siteUrl) {
+	if (siteUrl.startsWith(SC_DOMAIN_PREFIX)) return {
+		type: "INET_DOMAIN",
+		identifier: siteUrl.slice(10)
+	};
+	return {
+		type: "SITE",
+		identifier: siteUrl
+	};
+}
+function verificationMethodsFor(site) {
+	if (site.type === "INET_DOMAIN") return ["DNS_TXT", "DNS_CNAME"];
+	return [
+		"META",
+		"FILE",
+		"ANALYTICS",
+		"TAG_MANAGER"
+	];
+}
+async function getVerificationToken(client, siteUrl, method) {
+	const site = siteUrlToVerificationSite(siteUrl);
+	return {
+		...await client.verification.getToken({
+			site,
+			verificationMethod: method
+		}),
+		site
+	};
+}
+async function verifySite(client, siteUrl, method) {
+	const site = siteUrlToVerificationSite(siteUrl);
+	return client.verification.insert({
+		site,
+		verificationMethod: method
+	});
+}
+async function listVerifiedSites(client) {
+	return client.verification.list();
+}
 const MS_PER_DAY = 864e5;
 function toIsoDate(d) {
 	return d.toISOString().slice(0, 10);
@@ -377,11 +442,38 @@ function gscdumpApi(options) {
 				startRow += rows.length;
 			}
 		},
-		sites: async (opts) => {
-			return (await fetch("/api/sites", { signal: opts?.signal })).sites.map((s) => ({
-				siteUrl: s.gscSiteUrl,
-				permissionLevel: s.permissionLevel || "siteOwner"
-			}));
+		sites: (() => {
+			const list = async (opts) => {
+				return (await fetch("/api/sites", { signal: opts?.signal })).sites.map((s) => ({
+					siteUrl: s.gscSiteUrl,
+					permissionLevel: s.permissionLevel || "siteOwner"
+				}));
+			};
+			const unsupported = (op) => () => {
+				throw new Error(`sites.${op} not available via gscdump API. Use googleSearchConsole() with OAuth credentials.`);
+			};
+			return Object.assign(list, {
+				list,
+				add: unsupported("add"),
+				delete: unsupported("delete")
+			});
+		})(),
+		verification: {
+			getToken: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			insert: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			list: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			get: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			},
+			delete: () => {
+				throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
+			}
 		},
 		inspect: () => {
 			throw new Error("URL inspection not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
@@ -413,6 +505,7 @@ function gscdumpApi(options) {
 }
 const GSC_API = "https://searchconsole.googleapis.com";
 const INDEXING_API = "https://indexing.googleapis.com";
+const SITE_VERIFICATION_API = "https://www.googleapis.com/siteVerification/v1";
 function createAuth(options) {
 	let credentials = { refresh_token: options.refreshToken };
 	return {
@@ -529,8 +622,42 @@ function googleSearchConsole(auth, options = {}) {
 				startRow += rows.length;
 			}
 		},
-		sites: async (opts) => {
-			return (await fetch(`${GSC_API}/webmasters/v3/sites`, { signal: opts?.signal })).siteEntry || [];
+		sites: (() => {
+			const list = async (opts) => {
+				return (await fetch(`${GSC_API}/webmasters/v3/sites`, { signal: opts?.signal })).siteEntry || [];
+			};
+			return Object.assign(list, {
+				list,
+				add: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeURIComponent(siteUrl)}`, {
+					method: "PUT",
+					signal: opts?.signal
+				}),
+				delete: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeURIComponent(siteUrl)}`, {
+					method: "DELETE",
+					signal: opts?.signal
+				})
+			});
+		})(),
+		verification: {
+			getToken: (params, opts) => fetch(`${SITE_VERIFICATION_API}/token`, {
+				method: "POST",
+				body: params,
+				signal: opts?.signal
+			}),
+			insert: (params, opts) => fetch(`${SITE_VERIFICATION_API}/webResource`, {
+				method: "POST",
+				query: { verificationMethod: params.verificationMethod },
+				body: { site: params.site },
+				signal: opts?.signal
+			}),
+			list: async (opts) => {
+				return (await fetch(`${SITE_VERIFICATION_API}/webResource`, { signal: opts?.signal })).items || [];
+			},
+			get: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, { signal: opts?.signal }),
+			delete: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
+				method: "DELETE",
+				signal: opts?.signal
+			})
 		},
 		inspect: (siteUrl, url, opts) => fetch(`${GSC_API}/v1/urlInspection/index:inspect`, {
 			method: "POST",
@@ -796,4 +923,4 @@ function hasIndexingScope(scopes) {
 	if (!scopes) return false;
 	return scopes.includes("googleapis.com/auth/indexing");
 }
-export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, MS_PER_DAY, addDays, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, progressBar, requestIndexing, rowWithMetricDefaults, storageError, submitSitemap, toIsoDate };
+export { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_QUOTAS, GSC_RETENTION_MONTHS, INDEXING_DAILY_LIMIT, INDEXING_EFFECTIVE_LIMIT, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, MS_PER_DAY, addDays, addSite, batchInspectUrls, batchRequestIndexing, classifyError, countDays, createAuth, createFetch, daysAgo, deleteSite, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, formatErrorForCli, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingMetadata, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, googleSearchConsole, groupIntoRanges, gscdumpApi, hasGscReadScope, hasGscWriteScope, hasIndexingScope, inspectUrl, isPermissionDeniedError, isValidGscDate, listVerifiedSites, progressBar, requestIndexing, rowWithMetricDefaults, siteUrlToVerificationSite, storageError, submitSitemap, toIsoDate, verificationMethodsFor, verifySite };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gscdump",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "description": "Google Search Console API wrapper with typed query builder, streaming pagination, and SEO analysis functions",
   "author": {
     "name": "Harlan Wilton",
@@ -102,7 +102,7 @@
     "dayjs": "^1.11.20",
     "defu": "^6.1.7",
     "ofetch": "^1.5.1",
-    "ufo": "^1.6.3"
+    "ufo": "^1.6.4"
   },
   "devDependencies": {
     "@googleapis/indexing": "^6.0.1",