gscdump 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/api/batch.d.mts +12 -0
  2. package/dist/api/batch.mjs +29 -0
  3. package/dist/api/indexing.d.mts +37 -0
  4. package/dist/api/indexing.mjs +25 -0
  5. package/dist/api/inspection.d.mts +89 -0
  6. package/dist/api/inspection.mjs +128 -0
  7. package/dist/api/oauth.d.mts +48 -0
  8. package/dist/api/oauth.mjs +154 -0
  9. package/dist/api/sites.d.mts +45 -0
  10. package/dist/api/sites.mjs +34 -0
  11. package/dist/api/verification.d.mts +43 -0
  12. package/dist/api/verification.mjs +89 -0
  13. package/dist/contracts.d.mts +46 -1
  14. package/dist/core/cli-format.mjs +9 -0
  15. package/dist/core/client.d.mts +122 -0
  16. package/dist/core/client.mjs +246 -0
  17. package/dist/core/errors.d.mts +66 -0
  18. package/dist/core/errors.mjs +207 -0
  19. package/dist/core/indexing-issues.d.mts +60 -0
  20. package/dist/core/indexing-issues.mjs +139 -0
  21. package/dist/core/property.d.mts +41 -0
  22. package/dist/core/property.mjs +74 -0
  23. package/dist/core/quota.d.mts +2 -0
  24. package/dist/core/quota.mjs +2 -0
  25. package/dist/core/result.d.mts +19 -1
  26. package/dist/core/scope-values.d.mts +7 -0
  27. package/dist/core/scope-values.mjs +15 -0
  28. package/dist/core/scopes.d.mts +5 -0
  29. package/dist/core/scopes.mjs +11 -0
  30. package/dist/core/site-url.d.mts +25 -0
  31. package/dist/core/site-url.mjs +30 -0
  32. package/dist/core/types.d.mts +201 -0
  33. package/dist/core/types.mjs +1 -0
  34. package/dist/dates.d.mts +2 -2
  35. package/dist/dates.mjs +2 -2
  36. package/dist/index.d.mts +20 -814
  37. package/dist/index.mjs +19 -1243
  38. package/dist/onboarding.d.mts +2 -0
  39. package/dist/onboarding.mjs +2 -0
  40. package/dist/{_chunks → query}/builder.d.mts +2 -2
  41. package/dist/query/builder.mjs +86 -0
  42. package/dist/query/columns.d.mts +15 -0
  43. package/dist/query/columns.mjs +23 -0
  44. package/dist/query/constants.d.mts +19 -0
  45. package/dist/query/constants.mjs +16 -0
  46. package/dist/query/errors.d.mts +85 -0
  47. package/dist/query/errors.mjs +141 -0
  48. package/dist/query/index.d.mts +9 -75
  49. package/dist/query/index.mjs +7 -230
  50. package/dist/query/operator-meta.mjs +41 -0
  51. package/dist/query/operators.d.mts +24 -0
  52. package/dist/query/operators.mjs +124 -0
  53. package/dist/query/plan.d.mts +92 -2
  54. package/dist/query/plan.mjs +3 -1
  55. package/dist/query/resolver.d.mts +40 -0
  56. package/dist/query/resolver.mjs +243 -0
  57. package/dist/{_chunks → query}/types.d.mts +3 -25
  58. package/dist/query/utils/countries.d.mts +7 -0
  59. package/dist/{_chunks/resolver.mjs → query/utils/countries.mjs} +1 -434
  60. package/dist/{_chunks → query/utils}/dayjs.mjs +1 -1
  61. package/dist/sitemap.d.mts +26 -0
  62. package/dist/sitemap.mjs +70 -0
  63. package/dist/url.d.mts +9 -0
  64. package/dist/url.mjs +6 -0
  65. package/package.json +2 -2
  66. package/dist/_chunks/contracts.d.mts +0 -47
  67. package/dist/_chunks/plan.d.mts +0 -175
  68. package/dist/_chunks/result.d.mts +0 -20
  69. /package/dist/{_chunks → core}/gsc-dates.d.mts +0 -0
  70. /package/dist/{_chunks → core}/gsc-dates.mjs +0 -0
  71. /package/dist/{_chunks → query/utils}/dayjs.d.mts +0 -0
package/dist/index.mjs CHANGED
@@ -1,1247 +1,23 @@
1
+ import { runSequentialBatch } from "./api/batch.mjs";
2
+ import { batchRequestIndexing, getIndexingMetadata, requestIndexing } from "./api/indexing.mjs";
3
+ import { GSC_INDEXING_SCOPE, GSC_READ_SCOPE, GSC_SITE_VERIFICATION_SCOPE, GSC_WRITE_SCOPE, hasGoogleScope } from "./core/scope-values.mjs";
4
+ import { hasGscReadScope, hasGscWriteScope, hasIndexingScope } from "./core/scopes.mjs";
5
+ import { MAX_FLAT_INSPECTION_BATCH_CONCURRENCY, batchInspectUrls, batchInspectUrlsFlatSettled, canUseUrlInspection, getIndexingEligibility, getNextCheckAfter, getNextCheckPriority, inspectUrl, inspectUrlFlat } from "./api/inspection.mjs";
6
+ import { GscApiError, classifyError, gscErrorToException, isPermissionDeniedError, parseGoogleError, rethrowAsGscApiError } from "./core/errors.mjs";
1
7
  import { err, isErr, isOk, ok, unwrapResult } from "./core/result.mjs";
2
- import { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_RETENTION_MONTHS, MS_PER_DAY, addDays, countDays, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, groupIntoRanges, isValidGscDate, toIsoDate } from "./_chunks/gsc-dates.mjs";
3
- import { resolveToBody } from "./_chunks/resolver.mjs";
8
+ import { exchangeAuthCodeResult, introspectAccessToken, introspectAccessTokenResult, refreshAccessToken, revokeOAuthToken, revokeOAuthTokenResult } from "./api/oauth.mjs";
9
+ import { addSite, deleteSite, deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, submitSitemap } from "./api/sites.mjs";
10
+ import { getVerificationToken, getVerifiedSite, listVerifiedSites, resolveVerificationTarget, siteUrlToVerificationSite, unverifySite, verificationMethodsFor, verifySite } from "./api/verification.mjs";
11
+ import { DAYS_PER_RANGE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_RETENTION_MONTHS, MS_PER_DAY, addDays, countDays, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getLatestGscDate, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, groupIntoRanges, isValidGscDate, toIsoDate } from "./core/gsc-dates.mjs";
12
+ import { DEFAULT_GSC_REQUEST_TIMEOUT_MS, createAuth, createFetch, googleSearchConsole } from "./core/client.mjs";
13
+ import { INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, KNOWN_COVERAGE_STATES, KNOWN_PAGE_FETCH_STATES, unmappedInspectionReasons } from "./core/indexing-issues.mjs";
14
+ import { normalizeGscSiteUrl, normalizeRegistrationTarget, parseGscSiteUrl } from "./core/site-url.mjs";
15
+ import { findBestGscProperty, findExactGscProperty, formatGscPropertyCandidates, gscPropertyMatchesTarget, isVerifiedGscPermission, isVerifiedGscProperty, matchGscSite, pickBestGscProperty } from "./core/property.mjs";
16
+ import { URL_INSPECTION_EFFECTIVE_LIMIT } from "./core/quota.mjs";
17
+ import "./core/types.mjs";
4
18
  import { normalizeUrl } from "./normalize.mjs";
19
+ import { GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, accountNextActions, accountStatuses, analyticsNextActions, analyticsStatuses, hasOptionalIndexingScope, hasRequiredAnalyticsScope, indexingNextActions, indexingStatuses, lifecycleErrorCodes, parseGrantedScopes, propertyNextActions, propertyStatuses, querySourceModes, sitemapNextActions, sitemapStatuses } from "./onboarding.mjs";
20
+ import { discoverSitemap, fetchSitemapUrls } from "./sitemap.mjs";
5
21
  import { decodeSiteId, encodeSiteId, normalizeSiteUrl } from "./tenant.mjs";
6
- import { ofetch } from "ofetch";
7
- import { GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, accountNextActions, accountStatuses, analyticsNextActions, analyticsStatuses, hasOptionalIndexingScope, hasRequiredAnalyticsScope, indexingNextActions, indexingStatuses, lifecycleErrorCodes, parseGrantedScopes, propertyNextActions, propertyStatuses, querySourceModes, sitemapNextActions, sitemapStatuses } from "@gscdump/contracts/partner";
8
- async function runSequentialBatch(items, operation, options = {}) {
9
- const { delayMs = 0, concurrency = 1, onProgress } = options;
10
- const results = Array.from({ length: items.length });
11
- let completed = 0;
12
- if (concurrency <= 1) {
13
- for (let i = 0; i < items.length; i++) {
14
- const result = await operation(items[i], i);
15
- results[i] = result;
16
- onProgress?.(result, i, items.length);
17
- if (i < items.length - 1 && delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
18
- }
19
- return results;
20
- }
21
- const cursor = { i: 0 };
22
- const worker = async () => {
23
- while (true) {
24
- const i = cursor.i++;
25
- if (i >= items.length) return;
26
- const result = await operation(items[i], i);
27
- results[i] = result;
28
- completed++;
29
- onProgress?.(result, completed - 1, items.length);
30
- if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
31
- }
32
- };
33
- await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
34
- return results;
35
- }
36
- async function requestIndexing(client, url, options = {}) {
37
- const { type = "URL_UPDATED" } = options;
38
- return client.indexing.publish(url, type).then((r) => ({
39
- url,
40
- type,
41
- notifyTime: r.urlNotificationMetadata?.latestUpdate?.notifyTime || void 0
42
- }));
43
- }
44
- async function getIndexingMetadata(client, url) {
45
- return client.indexing.getMetadata(url).then((r) => ({
46
- url,
47
- latestUpdate: r.latestUpdate || void 0,
48
- latestRemove: r.latestRemove || void 0
49
- }));
50
- }
51
- async function batchRequestIndexing(client, urls, options = {}) {
52
- const { type = "URL_UPDATED", delayMs = 100, concurrency, onProgress } = options;
53
- return runSequentialBatch(urls, (url) => requestIndexing(client, url, { type }), {
54
- delayMs,
55
- concurrency,
56
- onProgress
57
- });
58
- }
59
- const GSC_READ_SCOPE = "https://www.googleapis.com/auth/webmasters.readonly";
60
- const GSC_WRITE_SCOPE = "https://www.googleapis.com/auth/webmasters";
61
- const GSC_INDEXING_SCOPE = "https://www.googleapis.com/auth/indexing";
62
- const GSC_SITE_VERIFICATION_SCOPE = "https://www.googleapis.com/auth/siteverification";
63
- function parseGoogleScopes(scopes) {
64
- if (!scopes) return [];
65
- if (typeof scopes === "string") return scopes.split(/\s+/).map((scope) => scope.trim()).filter(Boolean);
66
- return scopes.map((scope) => scope.trim()).filter(Boolean);
67
- }
68
- function hasGoogleScope(scopes, scope) {
69
- const tokens = parseGoogleScopes(scopes);
70
- const suffix = scope.replace("https://www.googleapis.com/auth/", "");
71
- return tokens.includes(scope) || tokens.includes(suffix);
72
- }
73
- function hasGscReadScope(scopes) {
74
- return hasGoogleScope(scopes, "https://www.googleapis.com/auth/webmasters.readonly") || hasGoogleScope(scopes, "https://www.googleapis.com/auth/webmasters");
75
- }
76
- function hasGscWriteScope(scopes) {
77
- return hasGoogleScope(scopes, GSC_WRITE_SCOPE);
78
- }
79
- function hasIndexingScope(scopes) {
80
- return hasGoogleScope(scopes, GSC_INDEXING_SCOPE);
81
- }
82
- async function inspectUrl(client, siteUrl, inspectionUrl) {
83
- const inspection = (await client.inspect(siteUrl, inspectionUrl)).inspectionResult;
84
- return {
85
- inspection,
86
- isIndexed: inspection?.indexStatusResult?.verdict === "PASS"
87
- };
88
- }
89
- async function batchInspectUrls(client, siteUrl, urls, options = {}) {
90
- const { delayMs = 200, concurrency, onProgress } = options;
91
- return runSequentialBatch(urls, async (url) => {
92
- const { inspection, isIndexed } = await inspectUrl(client, siteUrl, url);
93
- return {
94
- url,
95
- inspection,
96
- isIndexed
97
- };
98
- }, {
99
- delayMs,
100
- concurrency,
101
- onProgress
102
- });
103
- }
104
- async function inspectUrlFlat(client, siteUrl, inspectionUrl, options) {
105
- const inspection = (await client.inspect(siteUrl, inspectionUrl, options)).inspectionResult;
106
- const index = inspection?.indexStatusResult;
107
- const mobile = inspection?.mobileUsabilityResult;
108
- const rich = inspection?.richResultsResult;
109
- const amp = inspection?.ampResult;
110
- return {
111
- url: inspectionUrl,
112
- verdict: index?.verdict ?? null,
113
- coverageState: index?.coverageState ?? null,
114
- indexingState: index?.indexingState ?? null,
115
- robotsTxtState: index?.robotsTxtState ?? null,
116
- pageFetchState: index?.pageFetchState ?? null,
117
- lastCrawlTime: index?.lastCrawlTime ?? null,
118
- crawlingUserAgent: index?.crawledAs ?? null,
119
- userCanonical: index?.userCanonical ?? null,
120
- googleCanonical: index?.googleCanonical ?? null,
121
- sitemaps: index?.sitemap?.length ? JSON.stringify(index.sitemap) : null,
122
- referringUrls: index?.referringUrls?.length ? JSON.stringify(index.referringUrls) : null,
123
- mobileVerdict: mobile?.verdict ?? null,
124
- mobileIssues: mobile?.issues?.length ? JSON.stringify(mobile.issues) : null,
125
- richResultsVerdict: rich?.verdict ?? null,
126
- richResultsItems: rich?.detectedItems?.length ? JSON.stringify(rich.detectedItems) : null,
127
- ampVerdict: amp?.verdict ?? null,
128
- ampUrl: amp?.ampUrl ?? null,
129
- ampIndexingState: amp?.indexingState ?? null,
130
- ampIndexStatusVerdict: amp?.ampIndexStatusVerdict ?? null,
131
- ampRobotsTxtState: amp?.robotsTxtState ?? null,
132
- ampPageFetchState: amp?.pageFetchState ?? null,
133
- ampLastCrawlTime: amp?.lastCrawlTime ?? null,
134
- ampIssues: amp?.issues?.length ? JSON.stringify(amp.issues) : null,
135
- inspectionResultLink: inspection?.inspectionResultLink ?? null
136
- };
137
- }
138
- const MAX_FLAT_INSPECTION_BATCH_CONCURRENCY = 10;
139
- async function batchInspectUrlsFlatSettled(client, siteUrl, urls, options = {}) {
140
- const requestedConcurrency = Math.trunc(options.concurrency ?? 1);
141
- const concurrency = Number.isFinite(requestedConcurrency) ? Math.max(1, Math.min(requestedConcurrency, 10)) : 1;
142
- return runSequentialBatch([...urls], async (url) => {
143
- try {
144
- return {
145
- url,
146
- status: "fulfilled",
147
- value: await inspectUrlFlat(client, siteUrl, url, { signal: options.signal })
148
- };
149
- } catch (reason) {
150
- return {
151
- url,
152
- status: "rejected",
153
- reason
154
- };
155
- }
156
- }, {
157
- delayMs: options.delayMs ?? 200,
158
- concurrency,
159
- onProgress: options.onProgress
160
- });
161
- }
162
- function getNextCheckPriority(result) {
163
- if (!result.verdict || result.verdict === "VERDICT_UNSPECIFIED") return "medium";
164
- if (result.verdict === "FAIL" || result.verdict === "PARTIAL" || result.verdict === "NEUTRAL") return "high";
165
- if (result.verdict === "PASS") return "low";
166
- return "medium";
167
- }
168
- function getNextCheckAfter(priority) {
169
- const now = Math.floor(Date.now() / 1e3);
170
- switch (priority) {
171
- case "high": return now + 7 * 86400;
172
- case "medium": return now + 14 * 86400;
173
- case "low": return now + 30 * 86400;
174
- }
175
- }
176
- const INSPECTION_ALLOWED_PERMISSIONS = /* @__PURE__ */ new Set([
177
- "siteOwner",
178
- "siteFullUser",
179
- "siteRestrictedUser"
180
- ]);
181
- function canUseUrlInspection(permissionLevel) {
182
- return !!permissionLevel && INSPECTION_ALLOWED_PERMISSIONS.has(permissionLevel);
183
- }
184
- function grantedScopeList(scopes) {
185
- return scopes?.split(/\s+/).map((s) => s.trim()).filter(Boolean) ?? [];
186
- }
187
- function getIndexingEligibility(grantedScopes, permissionLevel) {
188
- const scopes = grantedScopeList(grantedScopes);
189
- if (!hasIndexingScope(grantedScopes)) return {
190
- indexingEligible: false,
191
- indexingIneligibleReason: "missing_indexing_scope",
192
- indexingPermissionLevel: permissionLevel ?? null,
193
- grantedScopes: scopes
194
- };
195
- if (!canUseUrlInspection(permissionLevel)) return {
196
- indexingEligible: false,
197
- indexingIneligibleReason: "insufficient_gsc_permission",
198
- indexingPermissionLevel: permissionLevel ?? null,
199
- grantedScopes: scopes
200
- };
201
- return {
202
- indexingEligible: true,
203
- indexingPermissionLevel: permissionLevel ?? null,
204
- grantedScopes: scopes
205
- };
206
- }
207
- function pickField(error, paths, is) {
208
- if (!error || typeof error !== "object") return void 0;
209
- for (const path of paths) {
210
- let current = error;
211
- for (const key of path) {
212
- if (!current || typeof current !== "object") {
213
- current = void 0;
214
- break;
215
- }
216
- current = current[key];
217
- }
218
- if (is(current)) return current;
219
- }
220
- }
221
- const isNumber = (v) => typeof v === "number";
222
- const isString = (v) => typeof v === "string";
223
- function extractStatus(error) {
224
- return pickField(error, [
225
- ["statusCode"],
226
- ["status"],
227
- ["response", "status"],
228
- ["code"]
229
- ], isNumber);
230
- }
231
- function extractMessage(error) {
232
- if (!error) return "Unknown error";
233
- if (typeof error === "string") return error;
234
- if (error instanceof Error) return error.message;
235
- if (typeof error !== "object") return String(error);
236
- return pickField(error, [
237
- [
238
- "data",
239
- "error",
240
- "message"
241
- ],
242
- ["message"],
243
- ["statusMessage"]
244
- ], isString) ?? String(error);
245
- }
246
- function extractRetryAfter(error) {
247
- const raw = pickField(error, [
248
- ["headers", "retry-after"],
249
- ["headers", "Retry-After"],
250
- [
251
- "response",
252
- "headers",
253
- "retry-after"
254
- ],
255
- [
256
- "response",
257
- "headers",
258
- "Retry-After"
259
- ]
260
- ], (v) => typeof v === "number" || typeof v === "string");
261
- if (typeof raw === "number") return raw;
262
- if (typeof raw === "string") {
263
- const seconds = Number.parseInt(raw, 10);
264
- return Number.isNaN(seconds) ? void 0 : seconds;
265
- }
266
- }
267
- const QUOTA_MESSAGE_RE = /quota|rate\s*limit/i;
268
- const QUOTA_REASONS = /* @__PURE__ */ new Set([
269
- "dailyLimitExceeded",
270
- "dailyLimitExceededUnreg",
271
- "rateLimitExceeded",
272
- "rateLimitExceededUnreg",
273
- "userRateLimitExceeded",
274
- "userRateLimitExceededUnreg",
275
- "quotaExceeded",
276
- "concurrentLimitExceeded",
277
- "variableTermLimitExceeded",
278
- "variableTermExpiredDailyExceeded",
279
- "servingLimitExceeded",
280
- "responseTooLarge",
281
- "limitExceeded",
282
- "batchSizeTooLarge",
283
- "RATE_LIMIT_EXCEEDED",
284
- "RESOURCE_EXHAUSTED"
285
- ]);
286
- function extractReason(cause) {
287
- const data = pickField(cause, [["data"]], (_v) => true);
288
- if (data) {
289
- const errorInfo = pickField(data, [["error", "details"]], (v) => Array.isArray(v));
290
- if (errorInfo) {
291
- const reason = errorInfo.find((d) => typeof d["@type"] === "string" && d["@type"].includes("ErrorInfo"))?.reason;
292
- if (typeof reason === "string") return reason;
293
- }
294
- const directErrors = pickField(data, [["error", "errors"]], (v) => Array.isArray(v));
295
- if (directErrors) {
296
- const reason = directErrors[0]?.reason;
297
- if (typeof reason === "string") return reason;
298
- }
299
- }
300
- }
301
- function isQuotaCondition(cause, message) {
302
- const reason = extractReason(cause);
303
- if (reason && QUOTA_REASONS.has(reason)) return true;
304
- return QUOTA_MESSAGE_RE.test(message);
305
- }
306
- function classifyError(cause) {
307
- const status = extractStatus(cause);
308
- const message = extractMessage(cause);
309
- if (status === 401) return {
310
- kind: "auth-expired",
311
- message,
312
- cause
313
- };
314
- if (status === 429) return {
315
- kind: "rate-limited",
316
- message,
317
- retryAfter: extractRetryAfter(cause),
318
- cause
319
- };
320
- if (status === 403) {
321
- if (isQuotaCondition(cause, message)) return {
322
- kind: "rate-limited",
323
- message,
324
- retryAfter: extractRetryAfter(cause),
325
- cause
326
- };
327
- return {
328
- kind: "auth-expired",
329
- message,
330
- cause
331
- };
332
- }
333
- if (status === 404 || status === 410) return {
334
- kind: "not-found",
335
- message,
336
- cause
337
- };
338
- if (status === 400 || status === 402 || status === 409 || status === 413 || status === 422) return {
339
- kind: "validation",
340
- message,
341
- cause
342
- };
343
- return {
344
- kind: "transport",
345
- message,
346
- status,
347
- cause
348
- };
349
- }
350
- function gscErrorToException(error) {
351
- const exception = new Error(error.message);
352
- if (error.cause !== void 0) exception.cause = error.cause;
353
- exception.gscError = error;
354
- return exception;
355
- }
356
- function parseGoogleError(text, httpStatus) {
357
- let parsed = null;
358
- try {
359
- parsed = JSON.parse(text);
360
- } catch (error) {
361
- if (!(error instanceof SyntaxError)) throw error;
362
- }
363
- if (!parsed || !("error" in parsed)) return {
364
- code: httpStatus ?? 500,
365
- message: text || "Unknown Google API error"
366
- };
367
- if (typeof parsed.error === "string") {
368
- const oauth = parsed;
369
- return {
370
- code: httpStatus ?? 400,
371
- message: oauth.error_description || oauth.error,
372
- reason: oauth.error
373
- };
374
- }
375
- const err = parsed.error;
376
- const errorInfo = err.details?.find((d) => d["@type"]?.includes("ErrorInfo"));
377
- return {
378
- code: err.code ?? httpStatus ?? 500,
379
- message: err.message || err.status || text || `HTTP ${httpStatus ?? "?"}`,
380
- reason: errorInfo?.reason,
381
- status: err.status
382
- };
383
- }
384
- var GscApiError = class extends Error {
385
- info;
386
- constructor(message, info) {
387
- super(message);
388
- this.info = info;
389
- this.name = "GscApiError";
390
- }
391
- };
392
- function rethrowAsGscApiError(prefix) {
393
- return (err) => {
394
- if (err instanceof GscApiError) throw err;
395
- const maybe = err;
396
- if (maybe && maybe.name === "FetchError") {
397
- const info = parseGoogleError(typeof maybe.data === "string" ? maybe.data : JSON.stringify(maybe.data ?? {}), maybe.statusCode);
398
- throw new GscApiError(`${prefix}: ${info.message}`, info);
399
- }
400
- throw err;
401
- };
402
- }
403
- const PERMISSION_SIGNALS = [
404
- "403 forbidden",
405
- "permission_denied",
406
- "does not have sufficient permission",
407
- "insufficient permission"
408
- ];
409
- function isPermissionDeniedError(err) {
410
- const msg = String(err?.message ?? err ?? "").toLowerCase();
411
- return PERMISSION_SIGNALS.some((s) => msg.includes(s));
412
- }
413
- const OAUTH_TOKEN_URL = "https://oauth2.googleapis.com/token";
414
- const OAUTH_TOKEN_INFO_URL = "https://oauth2.googleapis.com/tokeninfo";
415
- const OAUTH_REVOKE_URL = "https://oauth2.googleapis.com/revoke";
416
- const OAUTH_TIMEOUT_MS = 8e3;
417
- const OAUTH_MAX_ATTEMPTS = 3;
418
- const OAUTH_BACKOFF_MS = [
419
- 0,
420
- 400,
421
- 1500
422
- ];
423
- async function refreshAccessTokenResult(refreshToken, clientId, clientSecret) {
424
- const result = await postOAuthTokenResult(new URLSearchParams({
425
- client_id: clientId,
426
- client_secret: clientSecret,
427
- refresh_token: refreshToken,
428
- grant_type: "refresh_token"
429
- }), "refresh");
430
- if (!result.ok) return result;
431
- return ok({
432
- accessToken: result.value.accessToken,
433
- expiresAt: result.value.expiresAt,
434
- scope: result.value.scope
435
- });
436
- }
437
- async function refreshAccessToken(refreshToken, clientId, clientSecret) {
438
- return unwrapResult(await refreshAccessTokenResult(refreshToken, clientId, clientSecret), oauthErrorToException);
439
- }
440
- async function exchangeAuthCodeResult(code, clientId, clientSecret, redirectUri) {
441
- const result = await postOAuthTokenResult(new URLSearchParams({
442
- client_id: clientId,
443
- client_secret: clientSecret,
444
- code,
445
- redirect_uri: redirectUri,
446
- grant_type: "authorization_code"
447
- }), "exchange");
448
- if (!result.ok) return result;
449
- return result;
450
- }
451
- async function introspectAccessTokenResult(accessToken) {
452
- const response = await requestOAuthResult(`${OAUTH_TOKEN_INFO_URL}?access_token=${encodeURIComponent(accessToken)}`, { method: "GET" }, "introspect");
453
- if (!response.ok) return response;
454
- const parsed = await readOAuthJsonResult(response.value, "introspect");
455
- if (!parsed.ok) return parsed;
456
- const data = parsed.value;
457
- return ok({
458
- issuedTo: optionalString(data.issued_to),
459
- audience: optionalString(data.aud),
460
- authorizedParty: optionalString(data.azp),
461
- userId: optionalString(data.user_id),
462
- subject: optionalString(data.sub),
463
- scope: optionalString(data.scope),
464
- expiresIn: optionalNumber(data.expires_in),
465
- expiresAt: optionalNumber(data.exp),
466
- email: optionalString(data.email),
467
- emailVerified: optionalBoolean(data.email_verified),
468
- accessType: optionalString(data.access_type)
469
- });
470
- }
471
- async function introspectAccessToken(accessToken) {
472
- return unwrapResult(await introspectAccessTokenResult(accessToken), oauthErrorToException);
473
- }
474
- async function revokeOAuthTokenResult(token) {
475
- const response = await requestOAuthResult(OAUTH_REVOKE_URL, {
476
- method: "POST",
477
- headers: { "Content-Type": "application/x-www-form-urlencoded" },
478
- body: new URLSearchParams({ token })
479
- }, "revoke");
480
- return response.ok ? ok(void 0) : response;
481
- }
482
- async function revokeOAuthToken(token) {
483
- return unwrapResult(await revokeOAuthTokenResult(token), oauthErrorToException);
484
- }
485
- function oauthHttpError(op, info) {
486
- const message = `Failed to ${op} token: ${info.message}`;
487
- const cause = new GscApiError(message, info);
488
- return info.reason === "invalid_grant" || info.reason === "invalid_token" || info.code === 400 || info.code === 401 || info.code === 403 ? {
489
- kind: "auth-expired",
490
- message,
491
- cause
492
- } : {
493
- kind: "transport",
494
- message,
495
- status: info.code,
496
- cause
497
- };
498
- }
499
- function oauthErrorToException(error) {
500
- return error.cause instanceof Error ? error.cause : gscErrorToException(error);
501
- }
502
- async function postOAuthTokenResult(body, op) {
503
- const response = await requestOAuthResult(OAUTH_TOKEN_URL, {
504
- method: "POST",
505
- headers: { "Content-Type": "application/x-www-form-urlencoded" },
506
- body
507
- }, op);
508
- if (!response.ok) return response;
509
- const parsed = await readOAuthJsonResult(response.value, op);
510
- if (!parsed.ok) return parsed;
511
- const data = parsed.value;
512
- return ok({
513
- accessToken: data.access_token,
514
- expiresAt: Math.floor(Date.now() / 1e3) + data.expires_in,
515
- refreshToken: data.refresh_token,
516
- scope: data.scope
517
- });
518
- }
519
- async function requestOAuthResult(url, init, op) {
520
- let lastError;
521
- for (let attempt = 0; attempt < OAUTH_MAX_ATTEMPTS; attempt++) {
522
- if (OAUTH_BACKOFF_MS[attempt]) await new Promise((r) => setTimeout(r, OAUTH_BACKOFF_MS[attempt]));
523
- const res = await fetch(url, {
524
- ...init,
525
- signal: AbortSignal.timeout(OAUTH_TIMEOUT_MS)
526
- }).catch((error) => {
527
- lastError = error;
528
- return null;
529
- });
530
- if (!res) continue;
531
- if (!res.ok) return err(oauthHttpError(op, parseGoogleError(await res.text(), res.status)));
532
- return ok(res);
533
- }
534
- return err({
535
- kind: "transport",
536
- message: lastError instanceof Error ? lastError.message : `OAuth ${op} failed after ${OAUTH_MAX_ATTEMPTS} attempts`,
537
- cause: lastError
538
- });
539
- }
540
- async function readOAuthJsonResult(response, op) {
541
- try {
542
- return ok(await response.json());
543
- } catch (cause) {
544
- return err({
545
- kind: "transport",
546
- message: `Failed to parse ${op} token response`,
547
- status: response.status,
548
- cause
549
- });
550
- }
551
- }
552
- function optionalString(value) {
553
- return typeof value === "string" ? value : void 0;
554
- }
555
- function optionalNumber(value) {
556
- const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN;
557
- return Number.isFinite(parsed) ? parsed : void 0;
558
- }
559
- function optionalBoolean(value) {
560
- if (typeof value === "boolean") return value;
561
- if (value === "true") return true;
562
- if (value === "false") return false;
563
- }
564
- async function fetchSites(client) {
565
- return client.sites();
566
- }
567
- async function fetchSitesWithSitemaps(client, options = {}) {
568
- const sites = (await client.sites()).filter((s) => !!s.siteUrl && s.permissionLevel !== "siteUnverifiedUser");
569
- const requestedConcurrency = options.concurrency ?? 4;
570
- return runSequentialBatch(sites, async (site) => {
571
- const sitemaps = await client.sitemaps.list(site.siteUrl).catch(() => []);
572
- return {
573
- ...site,
574
- sitemaps
575
- };
576
- }, { concurrency: Number.isFinite(requestedConcurrency) ? Math.max(1, Math.floor(requestedConcurrency)) : 4 });
577
- }
578
- async function fetchSitemaps(client, siteUrl) {
579
- return client.sitemaps.list(siteUrl);
580
- }
581
- async function fetchSitemap(client, siteUrl, feedpath) {
582
- return client.sitemaps.get(siteUrl, feedpath);
583
- }
584
- async function submitSitemap(client, siteUrl, feedpath) {
585
- return client.sitemaps.submit(siteUrl, feedpath);
586
- }
587
- async function deleteSitemap(client, siteUrl, feedpath) {
588
- return client.sitemaps.delete(siteUrl, feedpath);
589
- }
590
- async function addSite(client, siteUrl) {
591
- return client.sites.add(siteUrl);
592
- }
593
- async function deleteSite(client, siteUrl) {
594
- return client.sites.delete(siteUrl);
595
- }
596
- const SC_DOMAIN_PREFIX = "sc-domain:";
597
- function bareDomain(siteUrl) {
598
- if (siteUrl.startsWith(SC_DOMAIN_PREFIX)) return siteUrl.slice(10);
599
- try {
600
- return new URL(siteUrl.startsWith("http") ? siteUrl : `https://${siteUrl}`).host;
601
- } catch {
602
- return siteUrl;
603
- }
604
- }
605
- function urlPrefix(siteUrl) {
606
- try {
607
- const url = new URL(siteUrl.startsWith("http") ? siteUrl : `https://${siteUrl}`);
608
- return `${url.protocol}//${url.host}/`;
609
- } catch {
610
- return siteUrl;
611
- }
612
- }
613
- function siteUrlToVerificationSite(siteUrl) {
614
- if (siteUrl.startsWith(SC_DOMAIN_PREFIX)) return {
615
- type: "INET_DOMAIN",
616
- identifier: siteUrl.slice(10)
617
- };
618
- return {
619
- type: "SITE",
620
- identifier: siteUrl
621
- };
622
- }
623
- function resolveVerificationTarget(siteUrl, method) {
624
- const isDomainProperty = siteUrl.startsWith(SC_DOMAIN_PREFIX);
625
- const chosen = method ?? (isDomainProperty ? "DNS_TXT" : "META");
626
- const isDns = chosen === "DNS_TXT" || chosen === "DNS_CNAME";
627
- if (isDomainProperty && !isDns) return {
628
- site: {
629
- type: "INET_DOMAIN",
630
- identifier: bareDomain(siteUrl)
631
- },
632
- method: "DNS_TXT"
633
- };
634
- if (isDns) return {
635
- site: {
636
- type: "INET_DOMAIN",
637
- identifier: bareDomain(siteUrl)
638
- },
639
- method: chosen
640
- };
641
- return {
642
- site: {
643
- type: "SITE",
644
- identifier: urlPrefix(siteUrl)
645
- },
646
- method: chosen
647
- };
648
- }
649
- function verificationMethodsFor(site) {
650
- if (site.type === "INET_DOMAIN") return ["DNS_TXT", "DNS_CNAME"];
651
- return [
652
- "META",
653
- "FILE",
654
- "ANALYTICS",
655
- "TAG_MANAGER"
656
- ];
657
- }
658
- async function getVerificationToken(client, siteUrl, method) {
659
- const { site, method: verificationMethod } = resolveVerificationTarget(siteUrl, method);
660
- return {
661
- ...await client.verification.getToken({
662
- site,
663
- verificationMethod
664
- }),
665
- site
666
- };
667
- }
668
- async function verifySite(client, siteUrl, method) {
669
- const { site, method: verificationMethod } = resolveVerificationTarget(siteUrl, method);
670
- return client.verification.insert({
671
- site,
672
- verificationMethod
673
- });
674
- }
675
- async function listVerifiedSites(client) {
676
- return client.verification.list();
677
- }
678
- async function getVerifiedSite(client, id) {
679
- return client.verification.get(id);
680
- }
681
- async function unverifySite(client, id) {
682
- return client.verification.delete(id);
683
- }
684
- function rowWithMetricDefaults(row) {
685
- return {
686
- clicks: row.clicks ?? 0,
687
- impressions: row.impressions ?? 0,
688
- ctr: row.ctr ?? 0,
689
- position: row.position ?? 0
690
- };
691
- }
692
- const GSC_API = "https://searchconsole.googleapis.com";
693
- const INDEXING_API = "https://indexing.googleapis.com";
694
- const SITE_VERIFICATION_API = "https://www.googleapis.com/siteVerification/v1";
695
- const DEFAULT_GSC_REQUEST_TIMEOUT_MS = 3e4;
696
- function encodeSiteUrl(siteUrl) {
697
- if (siteUrl.startsWith("sc-domain:")) return `sc-domain:${encodeURIComponent(siteUrl.slice(10))}`;
698
- return encodeURIComponent(siteUrl);
699
- }
700
- function assertValidSiteUrl(siteUrl) {
701
- if (siteUrl.startsWith("sc-domain:") && siteUrl.length > 10) return;
702
- if (/^https?:\/\/.+/.test(siteUrl)) return;
703
- throw new Error(`Invalid siteUrl: expected "https?://…" or "sc-domain:…", got "${siteUrl}"`);
704
- }
705
- const OAUTH_EXPIRY_SKEW_MS = 6e4;
706
- function createAuth(options) {
707
- let credentials = { refresh_token: options.refreshToken };
708
- let refreshPromise = null;
709
- return {
710
- get credentials() {
711
- return credentials;
712
- },
713
- async getAccessToken() {
714
- if (credentials?.access_token && credentials.expiry_date && credentials.expiry_date > Date.now() + OAUTH_EXPIRY_SKEW_MS) return { token: credentials.access_token };
715
- refreshPromise ??= refreshAccessToken(options.refreshToken, options.clientId, options.clientSecret).then((response) => {
716
- credentials = {
717
- ...credentials,
718
- access_token: response.accessToken,
719
- expiry_date: response.expiresAt * 1e3
720
- };
721
- return response.accessToken;
722
- }).finally(() => {
723
- refreshPromise = null;
724
- });
725
- return { token: await refreshPromise };
726
- }
727
- };
728
- }
729
- async function resolveToken(auth) {
730
- if (typeof auth === "string") return auth;
731
- if ("accessToken" in auth && typeof auth.accessToken === "string") return auth.accessToken;
732
- if ("getAccessToken" in auth && typeof auth.getAccessToken === "function") {
733
- const { token } = await auth.getAccessToken();
734
- return token || "";
735
- }
736
- if ("credentials" in auth && auth.credentials) return auth.credentials.access_token || "";
737
- return "";
738
- }
739
- function createFetch(auth, options) {
740
- const authState = typeof auth === "object" && auth !== null && "clientId" in auth && "refreshToken" in auth && !("getAccessToken" in auth) ? createAuth(auth) : auth;
741
- return ofetch.create({
742
- ...options,
743
- timeout: options?.timeout ?? 3e4,
744
- retry: 3,
745
- retryDelay: (ctx) => {
746
- const status = ctx.response?.status;
747
- if (status === 429 || status === 503) {
748
- const header = ctx.response?.headers.get("retry-after");
749
- if (header) {
750
- const secs = Number.parseInt(header, 10);
751
- if (Number.isFinite(secs)) return secs * 1e3;
752
- const when = Date.parse(header);
753
- if (Number.isFinite(when)) return Math.max(0, when - Date.now());
754
- }
755
- }
756
- return 1e3;
757
- },
758
- retryStatusCodes: [
759
- 408,
760
- 425,
761
- 429,
762
- 500,
763
- 502,
764
- 503,
765
- 504
766
- ],
767
- headers: {
768
- ...options?.headers,
769
- "Accept-Encoding": "gzip",
770
- "User-Agent": "gscdump (gzip)"
771
- },
772
- async onRequest({ options }) {
773
- const token = await resolveToken(authState);
774
- if (token) {
775
- options.headers = new Headers(options.headers);
776
- options.headers.set("Authorization", `Bearer ${token}`);
777
- }
778
- },
779
- async onResponseError(ctx) {
780
- if (ctx.response.status === 403) console.error("[gscdump] Permission denied (403). check your service account permissions being added to the GSC property.");
781
- if (options?.onResponseError) if (Array.isArray(options.onResponseError)) for (const handler of options.onResponseError) await handler(ctx);
782
- else await options.onResponseError(ctx);
783
- }
784
- });
785
- }
786
- function googleSearchConsole(auth, options = {}) {
787
- let fetch;
788
- const authState = typeof auth === "object" && auth !== null && "clientId" in auth && "refreshToken" in auth && !("getAccessToken" in auth) ? createAuth(auth) : auth;
789
- if (options.fetch) fetch = options.fetch;
790
- else {
791
- const fetchOptions = { ...options.fetchOptions };
792
- if (options.onRateLimited) {
793
- const originalOnError = fetchOptions.onResponseError;
794
- fetchOptions.onResponseError = async (ctx) => {
795
- if (ctx.response.status === 429) await options.onRateLimited({ response: ctx.response });
796
- if (originalOnError) if (Array.isArray(originalOnError)) for (const handler of originalOnError) await handler(ctx);
797
- else await originalOnError(ctx);
798
- };
799
- }
800
- fetch = createFetch(authState, fetchOptions);
801
- }
802
- const querySearchAnalytics = (siteUrl, body, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/searchAnalytics/query`, {
803
- method: "POST",
804
- body,
805
- signal: opts?.signal
806
- });
807
- return {
808
- async *query(siteUrl, builder, opts) {
809
- const state = builder.getState();
810
- const body = resolveToBody(state);
811
- const totalCap = body.rowLimit;
812
- const pageSize = Math.min(totalCap ?? 25e3, 25e3);
813
- let startRow = body.startRow || 0;
814
- let yielded = 0;
815
- let metadata;
816
- let responseAggregationType;
817
- while (true) {
818
- opts?.signal?.throwIfAborted();
819
- const remaining = totalCap ? totalCap - yielded : pageSize;
820
- if (remaining <= 0) break;
821
- const rowLimit = Math.min(pageSize, remaining);
822
- const response = await querySearchAnalytics(siteUrl, {
823
- ...body,
824
- startRow,
825
- rowLimit
826
- }, opts);
827
- if (response.metadata) metadata = response.metadata;
828
- if (response.responseAggregationType) responseAggregationType = response.responseAggregationType;
829
- const rows = (response.rows || []).map((row) => {
830
- const result = rowWithMetricDefaults(row);
831
- state.dimensions.forEach((dim, i) => {
832
- result[dim] = row.keys?.[i];
833
- });
834
- return result;
835
- });
836
- if (rows.length === 0) break;
837
- yield rows;
838
- yielded += rows.length;
839
- startRow += rows.length;
840
- }
841
- return {
842
- metadata,
843
- responseAggregationType
844
- };
845
- },
846
- sites: (() => {
847
- const list = async (opts) => {
848
- return (await fetch(`${GSC_API}/webmasters/v3/sites`, { signal: opts?.signal })).siteEntry || [];
849
- };
850
- return Object.assign(list, {
851
- list,
852
- get: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}`, { signal: opts?.signal }),
853
- add: (siteUrl, opts) => {
854
- assertValidSiteUrl(siteUrl);
855
- return fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}`, {
856
- method: "PUT",
857
- signal: opts?.signal
858
- });
859
- },
860
- delete: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}`, {
861
- method: "DELETE",
862
- signal: opts?.signal
863
- })
864
- });
865
- })(),
866
- verification: {
867
- getToken: (params, opts) => fetch(`${SITE_VERIFICATION_API}/token`, {
868
- method: "POST",
869
- body: params,
870
- signal: opts?.signal
871
- }),
872
- insert: (params, opts) => fetch(`${SITE_VERIFICATION_API}/webResource`, {
873
- method: "POST",
874
- query: { verificationMethod: params.verificationMethod },
875
- body: { site: params.site },
876
- signal: opts?.signal
877
- }),
878
- list: async (opts) => {
879
- return (await fetch(`${SITE_VERIFICATION_API}/webResource`, { signal: opts?.signal })).items || [];
880
- },
881
- get: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, { signal: opts?.signal }),
882
- delete: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
883
- method: "DELETE",
884
- signal: opts?.signal
885
- })
886
- },
887
- inspect: (siteUrl, url, opts) => fetch(`${GSC_API}/v1/urlInspection/index:inspect`, {
888
- method: "POST",
889
- body: opts?.languageCode ? {
890
- inspectionUrl: url,
891
- siteUrl,
892
- languageCode: opts.languageCode
893
- } : {
894
- inspectionUrl: url,
895
- siteUrl
896
- },
897
- signal: opts?.signal
898
- }),
899
- sitemaps: {
900
- list: async (siteUrl, opts) => {
901
- return (await fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps`, {
902
- signal: opts?.signal,
903
- query: opts?.sitemapIndex ? { sitemapIndex: opts.sitemapIndex } : void 0
904
- })).sitemap || [];
905
- },
906
- get: (siteUrl, feedpath, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`, { signal: opts?.signal }),
907
- submit: (siteUrl, feedpath, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`, {
908
- method: "PUT",
909
- signal: opts?.signal
910
- }),
911
- delete: (siteUrl, feedpath, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`, {
912
- method: "DELETE",
913
- signal: opts?.signal
914
- })
915
- },
916
- indexing: {
917
- publish: (url, type, opts) => fetch(`${INDEXING_API}/v3/urlNotifications:publish`, {
918
- method: "POST",
919
- body: {
920
- url,
921
- type
922
- },
923
- signal: opts?.signal
924
- }),
925
- getMetadata: (url, opts) => fetch(`${INDEXING_API}/v3/urlNotifications/metadata`, {
926
- query: { url },
927
- signal: opts?.signal
928
- })
929
- },
930
- searchAnalytics: { query: querySearchAnalytics }
931
- };
932
- }
933
- const INDEXING_ISSUE_FILTERS = {
934
- canonical_mismatch: `user_canonical IS NOT NULL AND google_canonical IS NOT NULL AND user_canonical != google_canonical`,
935
- stale_crawl: `last_crawl_time < datetime('now', '-30 days')`,
936
- very_stale_crawl: `last_crawl_time < datetime('now', '-60 days')`,
937
- not_indexed: `verdict IN ('FAIL', 'PARTIAL', 'NEUTRAL')`,
938
- unknown_to_google: `coverage_state = 'URL is unknown to Google'`,
939
- crawled_not_indexed: `coverage_state = 'Crawled - currently not indexed'`,
940
- discovered_not_indexed: `coverage_state = 'Discovered - currently not indexed'`,
941
- not_found: `page_fetch_state = 'NOT_FOUND' OR coverage_state = 'Not found (404)'`,
942
- soft_404: `page_fetch_state = 'SOFT_404' OR coverage_state = 'Soft 404'`,
943
- server_error: `page_fetch_state = 'SERVER_ERROR' OR coverage_state = 'Server error (5xx)'`,
944
- access_forbidden: `page_fetch_state = 'ACCESS_FORBIDDEN' OR coverage_state = 'Blocked due to access forbidden (403)'`,
945
- access_denied: `page_fetch_state = 'ACCESS_DENIED' OR coverage_state = 'Blocked due to unauthorized request (401)'`,
946
- blocked_4xx: `page_fetch_state = 'BLOCKED_4XX' OR coverage_state = 'Blocked due to other 4xx issue'`,
947
- redirect_error: `page_fetch_state = 'REDIRECT_ERROR' OR coverage_state = 'Redirect error'`,
948
- crawl_error: `page_fetch_state IN ('INTERNAL_CRAWL_ERROR', 'INVALID_URL')`,
949
- blocked_robots: `robots_txt_state = 'DISALLOWED'`,
950
- noindex: `indexing_state LIKE '%noindex%' OR coverage_state LIKE '%noindex%'`,
951
- redirect: `coverage_state = 'Page with redirect'`,
952
- sitemap_redirect: `coverage_state = 'Page with redirect' AND sitemaps IS NOT NULL AND sitemaps != '[]'`,
953
- alternate_canonical: `coverage_state = 'Alternate page with proper canonical tag'`,
954
- duplicate_no_canonical: `coverage_state = 'Duplicate without user-selected canonical'`,
955
- page_removed: `coverage_state = 'Blocked by page removal tool'`,
956
- fragment_url: `url LIKE '%#%'`,
957
- mobile_fail: `mobile_verdict IN ('FAIL', 'PARTIAL')`,
958
- rich_results_fail: `rich_results_verdict = 'FAIL'`,
959
- rich_results_warning: `rich_results_verdict = 'PARTIAL'`,
960
- rich_results_pass: `rich_results_verdict = 'PASS'`
961
- };
962
- const INDEXING_ISSUE_LABELS = {
963
- canonical_mismatch: "Canonical mismatch",
964
- stale_crawl: "Not crawled in 30+ days",
965
- very_stale_crawl: "Not crawled in 60+ days",
966
- not_indexed: "Not indexed",
967
- unknown_to_google: "Unknown to Google",
968
- crawled_not_indexed: "Crawled but not indexed",
969
- discovered_not_indexed: "Discovered but not indexed",
970
- not_found: "404 Not Found",
971
- soft_404: "Soft 404",
972
- server_error: "Server error",
973
- access_forbidden: "Blocked: forbidden (403)",
974
- access_denied: "Blocked: unauthorized (401)",
975
- blocked_4xx: "Blocked: other 4xx",
976
- redirect_error: "Redirect error",
977
- crawl_error: "Crawl error",
978
- blocked_robots: "Blocked by robots.txt",
979
- noindex: "Noindex tag",
980
- redirect: "Redirect",
981
- sitemap_redirect: "Sitemap URL redirects",
982
- alternate_canonical: "Alternate page with canonical",
983
- duplicate_no_canonical: "Duplicate, no canonical declared",
984
- page_removed: "Removed via removal tool",
985
- fragment_url: "Fragment URL (#)",
986
- mobile_fail: "Mobile usability issues",
987
- rich_results_fail: "Rich results errors",
988
- rich_results_warning: "Rich results warnings",
989
- rich_results_pass: "Has rich results"
990
- };
991
- const INDEXING_ISSUE_SEVERITY = {
992
- canonical_mismatch: "warning",
993
- stale_crawl: "info",
994
- very_stale_crawl: "warning",
995
- not_indexed: "error",
996
- unknown_to_google: "warning",
997
- crawled_not_indexed: "error",
998
- discovered_not_indexed: "warning",
999
- not_found: "error",
1000
- soft_404: "error",
1001
- server_error: "error",
1002
- access_forbidden: "error",
1003
- access_denied: "error",
1004
- blocked_4xx: "error",
1005
- redirect_error: "error",
1006
- crawl_error: "warning",
1007
- blocked_robots: "warning",
1008
- noindex: "info",
1009
- redirect: "info",
1010
- sitemap_redirect: "warning",
1011
- alternate_canonical: "info",
1012
- duplicate_no_canonical: "warning",
1013
- page_removed: "info",
1014
- fragment_url: "warning",
1015
- mobile_fail: "warning",
1016
- rich_results_fail: "error",
1017
- rich_results_warning: "warning",
1018
- rich_results_pass: "info"
1019
- };
1020
- const KNOWN_PAGE_FETCH_STATES = /* @__PURE__ */ new Set([
1021
- "PAGE_FETCH_STATE_UNSPECIFIED",
1022
- "SUCCESSFUL",
1023
- "SOFT_404",
1024
- "BLOCKED_ROBOTS_TXT",
1025
- "NOT_FOUND",
1026
- "ACCESS_DENIED",
1027
- "SERVER_ERROR",
1028
- "REDIRECT_ERROR",
1029
- "ACCESS_FORBIDDEN",
1030
- "BLOCKED_4XX",
1031
- "INTERNAL_CRAWL_ERROR",
1032
- "INVALID_URL"
1033
- ]);
1034
- const KNOWN_COVERAGE_STATES = /* @__PURE__ */ new Set([
1035
- "Submitted and indexed",
1036
- "Indexed, not submitted in sitemap",
1037
- "Indexed, though blocked by robots.txt",
1038
- "Indexed; consider marking as canonical",
1039
- "URL is unknown to Google",
1040
- "Crawled - currently not indexed",
1041
- "Discovered - currently not indexed",
1042
- "Not found (404)",
1043
- "Soft 404",
1044
- "Server error (5xx)",
1045
- "Blocked due to access forbidden (403)",
1046
- "Blocked due to unauthorized request (401)",
1047
- "Blocked due to other 4xx issue",
1048
- "Redirect error",
1049
- "Blocked by robots.txt",
1050
- "Excluded by ‘noindex’ tag",
1051
- "Page with redirect",
1052
- "Alternate page with proper canonical tag",
1053
- "Duplicate without user-selected canonical",
1054
- "Duplicate, Google chose different canonical than user",
1055
- "Blocked by page removal tool"
1056
- ]);
1057
- function unmappedInspectionReasons(rows) {
1058
- const coverageStates = /* @__PURE__ */ new Set();
1059
- const pageFetchStates = /* @__PURE__ */ new Set();
1060
- for (const row of rows) {
1061
- const coverage = row.coverageState?.trim();
1062
- if (coverage && !KNOWN_COVERAGE_STATES.has(coverage)) coverageStates.add(coverage);
1063
- const fetchState = row.pageFetchState?.trim();
1064
- if (fetchState && !KNOWN_PAGE_FETCH_STATES.has(fetchState)) pageFetchStates.add(fetchState);
1065
- }
1066
- return {
1067
- coverageStates: [...coverageStates],
1068
- pageFetchStates: [...pageFetchStates]
1069
- };
1070
- }
1071
- const SCHEME_RE = /^(sc-domain:|https?:\/\/)/;
1072
- function parseGscSiteUrl(siteUrl) {
1073
- const isDomain = siteUrl.startsWith("sc-domain:");
1074
- let hostname = siteUrl;
1075
- if (isDomain) hostname = siteUrl.slice(10);
1076
- else try {
1077
- hostname = new URL(siteUrl).hostname;
1078
- } catch {
1079
- hostname = siteUrl;
1080
- }
1081
- const displayLabel = siteUrl.replace(SCHEME_RE, "").replace(/\/$/, "");
1082
- return {
1083
- label: siteUrl,
1084
- hostname,
1085
- displayLabel,
1086
- propertyType: isDomain ? "domain" : "url-prefix",
1087
- isDomain
1088
- };
1089
- }
1090
- function normalizeGscSiteUrl(siteUrl) {
1091
- return siteUrl.replace(/^sc-domain:/, "").replace(/^https?:\/\//, "").replace(/^www\./, "").replace(/\/$/, "").toLowerCase();
1092
- }
1093
- function normalizeRegistrationTarget(inputUrl) {
1094
- const trimmed = inputUrl.trim();
1095
- if (!trimmed) return null;
1096
- const inputParsed = parseGscSiteUrl(trimmed);
1097
- if (inputParsed.isDomain) return inputParsed.hostname.toLowerCase();
1098
- return trimmed.match(/^(?:https?:\/\/)?([^/]+)/)?.[1]?.toLowerCase() ?? null;
1099
- }
1100
- const VERIFIED_PERMISSIONS = /* @__PURE__ */ new Set([
1101
- "siteOwner",
1102
- "siteFullUser",
1103
- "siteRestrictedUser"
1104
- ]);
1105
- function isVerifiedGscProperty(property) {
1106
- return !!property?.permissionLevel && VERIFIED_PERMISSIONS.has(property.permissionLevel);
1107
- }
1108
- function isVerifiedGscPermission(level) {
1109
- return !!level && VERIFIED_PERMISSIONS.has(level);
1110
- }
1111
- const stripWww = (d) => d.replace(/^www\./, "");
1112
- function gscPropertyMatchesTarget(targetDomain, propertyUrl) {
1113
- if (!propertyUrl) return false;
1114
- const cleanTarget = stripWww(targetDomain);
1115
- const parsed = parseGscSiteUrl(propertyUrl);
1116
- if (parsed.isDomain) return cleanTarget === parsed.hostname || cleanTarget.endsWith(`.${parsed.hostname}`);
1117
- const match = propertyUrl.match(/^https?:\/\/([^/]+)/);
1118
- return !!match && stripWww(match[1].toLowerCase()) === cleanTarget;
1119
- }
1120
- function matchGscSite(siteUrl, gscSiteUrl) {
1121
- if (!siteUrl || !gscSiteUrl) return false;
1122
- const getHostname = (url) => {
1123
- if (url.startsWith("sc-domain:")) return url.replace("sc-domain:", "");
1124
- try {
1125
- return new URL(url).hostname;
1126
- } catch {
1127
- return url;
1128
- }
1129
- };
1130
- return gscPropertyMatchesTarget(getHostname(siteUrl), gscSiteUrl);
1131
- }
1132
- function pickBestGscProperty(origin, availableSites) {
1133
- const matches = availableSites.filter((p) => matchGscSite(origin, p.siteUrl));
1134
- if (!matches.length) return void 0;
1135
- const isDomain = (p) => !!p.siteUrl?.startsWith("sc-domain:");
1136
- const isHttps = (p) => !!p.siteUrl?.startsWith("https://");
1137
- const pickTier = (pool) => pool.find(isDomain) ?? pool.find(isHttps) ?? pool[0];
1138
- const verified = matches.filter((p) => isVerifiedGscPermission(p.permissionLevel));
1139
- const pool = verified.length ? verified : matches;
1140
- const originHost = stripWww(parseGscSiteUrl(origin).hostname.toLowerCase());
1141
- const exact = pool.filter((property) => {
1142
- if (!property.siteUrl) return false;
1143
- return stripWww(parseGscSiteUrl(property.siteUrl).hostname.toLowerCase()) === originHost;
1144
- });
1145
- return pickTier(exact.length ? exact : pool);
1146
- }
1147
- function findBestGscProperty(targetDomain, properties) {
1148
- const cleanTarget = stripWww(targetDomain);
1149
- const domainProperty = properties.find((property) => {
1150
- if (!property.siteUrl) return false;
1151
- const parsed = parseGscSiteUrl(property.siteUrl);
1152
- if (!parsed.isDomain) return false;
1153
- return cleanTarget === parsed.hostname || cleanTarget.endsWith(`.${parsed.hostname}`);
1154
- });
1155
- const urlProperty = properties.find((property) => {
1156
- if (!property.siteUrl) return false;
1157
- const match = property.siteUrl.match(/^https?:\/\/([^/]+)/);
1158
- return match && stripWww(match[1].toLowerCase()) === cleanTarget;
1159
- });
1160
- return {
1161
- matchedSite: (isVerifiedGscProperty(domainProperty) ? domainProperty : isVerifiedGscProperty(urlProperty) ? urlProperty : domainProperty || urlProperty) ?? null,
1162
- domainProperty: domainProperty ?? null,
1163
- urlProperty: urlProperty ?? null
1164
- };
1165
- }
1166
- function findExactGscProperty(propertyUrl, properties) {
1167
- return properties.find((property) => property.siteUrl === propertyUrl) ?? null;
1168
- }
1169
- function formatGscPropertyCandidates(candidates) {
1170
- return candidates.filter((c) => !!c).map((property) => `${property.siteUrl} (${property.permissionLevel})`).join(", ");
1171
- }
1172
- const URL_INSPECTION_EFFECTIVE_LIMIT = 1800;
1173
- const FETCH_TIMEOUT_MS = 1e4;
1174
- const COMMON_PATHS = ["/sitemap.xml", "/sitemap_index.xml"];
1175
- const SITEMAP_DIRECTIVE_RE = /^Sitemap:\s*(\S+)/im;
1176
- async function discoverSitemap(domain, options = {}) {
1177
- const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
1178
- const baseUrl = `https://${domain}`;
1179
- const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
1180
- const robotsRes = await fetch(`${baseUrl}/robots.txt`, {
1181
- headers: { "User-Agent": userAgent },
1182
- signal: signalFor()
1183
- }).catch(() => null);
1184
- if (robotsRes?.ok) {
1185
- const match = (await robotsRes.text()).match(SITEMAP_DIRECTIVE_RE);
1186
- if (match?.[1]) {
1187
- if ((await fetch(match[1], {
1188
- method: "HEAD",
1189
- signal: signalFor()
1190
- }).catch(() => null))?.ok) return match[1];
1191
- }
1192
- }
1193
- for (const path of COMMON_PATHS) {
1194
- const url = `${baseUrl}${path}`;
1195
- if ((await fetch(url, {
1196
- method: "HEAD",
1197
- headers: { "User-Agent": userAgent },
1198
- signal: signalFor()
1199
- }).catch(() => null))?.ok) return url;
1200
- }
1201
- return null;
1202
- }
1203
- const LOC_RE = /<loc>([^<]+)<\/loc>/gi;
1204
- const SITEMAPINDEX_RE = /<sitemapindex\b/i;
1205
- async function fetchSitemapUrls(sitemapUrl, options = {}) {
1206
- const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
1207
- const maxDepth = options.maxDepth ?? 3;
1208
- const limit = options.limit;
1209
- const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
1210
- const seen = /* @__PURE__ */ new Set();
1211
- const out = [];
1212
- const visit = async (url, depth) => {
1213
- if (limit != null && out.length >= limit) return;
1214
- if (depth > maxDepth) return;
1215
- const res = await fetch(url, {
1216
- headers: { "User-Agent": userAgent },
1217
- signal: signalFor()
1218
- });
1219
- if (!res.ok) throw new Error(`Fetch ${url} failed: ${res.status}`);
1220
- const text = await res.text();
1221
- if (SITEMAPINDEX_RE.test(text)) {
1222
- for (const match of text.matchAll(LOC_RE)) {
1223
- if (limit != null && out.length >= limit) return;
1224
- const child = match[1].trim();
1225
- if (!child) continue;
1226
- await visit(child, depth + 1);
1227
- }
1228
- return;
1229
- }
1230
- for (const match of text.matchAll(LOC_RE)) {
1231
- const u = match[1].trim();
1232
- if (!u) continue;
1233
- if (seen.has(u)) continue;
1234
- seen.add(u);
1235
- out.push(u);
1236
- if (limit != null && out.length >= limit) return;
1237
- }
1238
- };
1239
- await visit(sitemapUrl, 0);
1240
- return out;
1241
- }
1242
- function urlMatchKey(url) {
1243
- const u = URL.parse(url);
1244
- if (!u) return null;
1245
- return [u.hostname.replace(/^www\./, ""), u.pathname.replace(/\/$/, "") || "/"].join("").toLowerCase();
1246
- }
22
+ import { urlMatchKey } from "./url.mjs";
1247
23
  export { DAYS_PER_RANGE, DEFAULT_GSC_REQUEST_TIMEOUT_MS, GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_INDEXING_SCOPE, GSC_READ_SCOPE, GSC_RETENTION_MONTHS, GSC_SITE_VERIFICATION_SCOPE, GSC_WRITE_SCOPE, GscApiError, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, KNOWN_COVERAGE_STATES, KNOWN_PAGE_FETCH_STATES, MAX_FLAT_INSPECTION_BATCH_CONCURRENCY, MS_PER_DAY, URL_INSPECTION_EFFECTIVE_LIMIT, accountNextActions, accountStatuses, addDays, addSite, analyticsNextActions, analyticsStatuses, batchInspectUrls, batchInspectUrlsFlatSettled, batchRequestIndexing, canUseUrlInspection, classifyError, countDays, createAuth, createFetch, decodeSiteId, deleteSite, deleteSitemap, discoverSitemap, encodeSiteId, err, exchangeAuthCodeResult, fetchSitemap, fetchSitemapUrls, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, findBestGscProperty, findExactGscProperty, formatGscPropertyCandidates, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingEligibility, getIndexingMetadata, getLatestGscDate, getNextCheckAfter, getNextCheckPriority, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, getVerifiedSite, googleSearchConsole, groupIntoRanges, gscErrorToException, gscPropertyMatchesTarget, hasGoogleScope, hasGscReadScope, hasGscWriteScope, hasIndexingScope, hasOptionalIndexingScope, hasRequiredAnalyticsScope, indexingNextActions, indexingStatuses, inspectUrl, inspectUrlFlat, introspectAccessToken, introspectAccessTokenResult, isErr, isOk, isPermissionDeniedError, isValidGscDate, isVerifiedGscPermission, isVerifiedGscProperty, lifecycleErrorCodes, listVerifiedSites, matchGscSite, normalizeGscSiteUrl, normalizeRegistrationTarget, normalizeSiteUrl, normalizeUrl, ok, parseGoogleError, parseGrantedScopes, parseGscSiteUrl, pickBestGscProperty, propertyNextActions, propertyStatuses, querySourceModes, refreshAccessToken, requestIndexing, resolveVerificationTarget, rethrowAsGscApiError, revokeOAuthToken, revokeOAuthTokenResult, runSequentialBatch, siteUrlToVerificationSite, sitemapNextActions, sitemapStatuses, submitSitemap, toIsoDate, unmappedInspectionReasons, unverifySite, unwrapResult, urlMatchKey, verificationMethodsFor, verifySite };