gscdump 0.40.2 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -123
- package/dist/core/result.d.mts +1 -10
- package/dist/core/result.mjs +1 -10
- package/dist/index.d.mts +212 -87
- package/dist/index.mjs +132 -233
- package/dist/query/index.d.mts +11 -8
- package/dist/query/index.mjs +33 -22
- package/dist/query/plan.d.mts +4 -1
- package/dist/query/plan.mjs +6 -0
- package/package.json +4 -25
- package/dist/api/index.d.mts +0 -748
- package/dist/api/index.mjs +0 -2989
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ofetch } from "ofetch";
|
|
2
|
-
import { GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, accountNextActions, accountStatuses, analyticsNextActions, analyticsStatuses, hasOptionalIndexingScope, hasRequiredAnalyticsScope, indexingNextActions, indexingStatuses, lifecycleErrorCodes,
|
|
2
|
+
import { GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, accountNextActions, accountStatuses, analyticsNextActions, analyticsStatuses, hasOptionalIndexingScope, hasRequiredAnalyticsScope, indexingNextActions, indexingStatuses, lifecycleErrorCodes, parseGrantedScopes, propertyNextActions, propertyStatuses, querySourceModes, sitemapNextActions, sitemapStatuses } from "@gscdump/contracts/partner";
|
|
3
3
|
async function runSequentialBatch(items, operation, options = {}) {
|
|
4
4
|
const { delayMs = 0, concurrency = 1, onProgress } = options;
|
|
5
5
|
const results = Array.from({ length: items.length });
|
|
@@ -96,8 +96,8 @@ async function batchInspectUrls(client, siteUrl, urls, options = {}) {
|
|
|
96
96
|
onProgress
|
|
97
97
|
});
|
|
98
98
|
}
|
|
99
|
-
async function inspectUrlFlat(client, siteUrl, inspectionUrl) {
|
|
100
|
-
const inspection = (await client.inspect(siteUrl, inspectionUrl)).inspectionResult;
|
|
99
|
+
async function inspectUrlFlat(client, siteUrl, inspectionUrl, options) {
|
|
100
|
+
const inspection = (await client.inspect(siteUrl, inspectionUrl, options)).inspectionResult;
|
|
101
101
|
const index = inspection?.indexStatusResult;
|
|
102
102
|
const mobile = inspection?.mobileUsabilityResult;
|
|
103
103
|
const rich = inspection?.richResultsResult;
|
|
@@ -130,6 +130,30 @@ async function inspectUrlFlat(client, siteUrl, inspectionUrl) {
|
|
|
130
130
|
inspectionResultLink: inspection?.inspectionResultLink ?? null
|
|
131
131
|
};
|
|
132
132
|
}
|
|
133
|
+
const MAX_FLAT_INSPECTION_BATCH_CONCURRENCY = 10;
|
|
134
|
+
async function batchInspectUrlsFlatSettled(client, siteUrl, urls, options = {}) {
|
|
135
|
+
const requestedConcurrency = Math.trunc(options.concurrency ?? 1);
|
|
136
|
+
const concurrency = Number.isFinite(requestedConcurrency) ? Math.max(1, Math.min(requestedConcurrency, 10)) : 1;
|
|
137
|
+
return runSequentialBatch([...urls], async (url) => {
|
|
138
|
+
try {
|
|
139
|
+
return {
|
|
140
|
+
url,
|
|
141
|
+
status: "fulfilled",
|
|
142
|
+
value: await inspectUrlFlat(client, siteUrl, url, { signal: options.signal })
|
|
143
|
+
};
|
|
144
|
+
} catch (reason) {
|
|
145
|
+
return {
|
|
146
|
+
url,
|
|
147
|
+
status: "rejected",
|
|
148
|
+
reason
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
}, {
|
|
152
|
+
delayMs: options.delayMs ?? 200,
|
|
153
|
+
concurrency,
|
|
154
|
+
onProgress: options.onProgress
|
|
155
|
+
});
|
|
156
|
+
}
|
|
133
157
|
function getNextCheckPriority(result) {
|
|
134
158
|
if (!result.verdict || result.verdict === "VERDICT_UNSPECIFIED") return "medium";
|
|
135
159
|
if (result.verdict === "FAIL" || result.verdict === "PARTIAL" || result.verdict === "NEUTRAL") return "high";
|
|
@@ -175,11 +199,6 @@ function getIndexingEligibility(grantedScopes, permissionLevel) {
|
|
|
175
199
|
grantedScopes: scopes
|
|
176
200
|
};
|
|
177
201
|
}
|
|
178
|
-
const GSC_QUOTAS = {
|
|
179
|
-
searchAnalytics: 25e3,
|
|
180
|
-
urlInspection: 2e3,
|
|
181
|
-
indexing: 200
|
|
182
|
-
};
|
|
183
202
|
function pickField(error, paths, is) {
|
|
184
203
|
if (!error || typeof error !== "object") return void 0;
|
|
185
204
|
for (const path of paths) {
|
|
@@ -323,13 +342,6 @@ function classifyError(cause) {
|
|
|
323
342
|
cause
|
|
324
343
|
};
|
|
325
344
|
}
|
|
326
|
-
function storageError(message, cause) {
|
|
327
|
-
return {
|
|
328
|
-
kind: "storage",
|
|
329
|
-
message,
|
|
330
|
-
cause
|
|
331
|
-
};
|
|
332
|
-
}
|
|
333
345
|
function gscErrorToException(error) {
|
|
334
346
|
const exception = new Error(error.message);
|
|
335
347
|
if (error.cause !== void 0) exception.cause = error.cause;
|
|
@@ -393,33 +405,6 @@ function isPermissionDeniedError(err) {
|
|
|
393
405
|
const msg = String(err?.message ?? err ?? "").toLowerCase();
|
|
394
406
|
return PERMISSION_SIGNALS.some((s) => msg.includes(s));
|
|
395
407
|
}
|
|
396
|
-
function suggestionFor(err) {
|
|
397
|
-
switch (err.kind) {
|
|
398
|
-
case "auth-expired": return "Run `gscdump auth` to re-authenticate.";
|
|
399
|
-
case "rate-limited": {
|
|
400
|
-
const retryIn = err.retryAfter ? `${err.retryAfter}s` : "a few minutes";
|
|
401
|
-
if (QUOTA_MESSAGE_RE.test(err.message)) {
|
|
402
|
-
if (err.message.includes("Indexing API")) return `Indexing API quota exhausted (~${GSC_QUOTAS.indexing}/day). Try again tomorrow.`;
|
|
403
|
-
return `Quota or rate limit hit (Search Analytics ~${GSC_QUOTAS.searchAnalytics}/day). Try again in ${retryIn}.`;
|
|
404
|
-
}
|
|
405
|
-
return `Rate limited. Slow down requests. Try again in ${retryIn}.`;
|
|
406
|
-
}
|
|
407
|
-
case "not-found":
|
|
408
|
-
case "validation":
|
|
409
|
-
case "storage":
|
|
410
|
-
case "transport": return "";
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
function formatErrorForCli(cause) {
|
|
414
|
-
const err = classifyError(cause);
|
|
415
|
-
const lines = [`\x1B[31m${err.message}\x1B[0m`];
|
|
416
|
-
const suggestion = suggestionFor(err);
|
|
417
|
-
if (suggestion) {
|
|
418
|
-
lines.push("");
|
|
419
|
-
lines.push(suggestion);
|
|
420
|
-
}
|
|
421
|
-
return lines.join("\n");
|
|
422
|
-
}
|
|
423
408
|
function ok(value) {
|
|
424
409
|
return {
|
|
425
410
|
ok: true,
|
|
@@ -438,20 +423,13 @@ function isOk(result) {
|
|
|
438
423
|
function isErr(result) {
|
|
439
424
|
return !result.ok;
|
|
440
425
|
}
|
|
441
|
-
function mapResult(result, f) {
|
|
442
|
-
return result.ok ? ok(f(result.value)) : result;
|
|
443
|
-
}
|
|
444
|
-
function mapErr(result, f) {
|
|
445
|
-
return result.ok ? result : err(f(result.error));
|
|
446
|
-
}
|
|
447
|
-
function matchResult(result, handlers) {
|
|
448
|
-
return result.ok ? handlers.onOk(result.value) : handlers.onErr(result.error);
|
|
449
|
-
}
|
|
450
426
|
function unwrapResult(result, toError) {
|
|
451
427
|
if (result.ok) return result.value;
|
|
452
428
|
throw toError(result.error);
|
|
453
429
|
}
|
|
454
430
|
const OAUTH_TOKEN_URL = "https://oauth2.googleapis.com/token";
|
|
431
|
+
const OAUTH_TOKEN_INFO_URL = "https://oauth2.googleapis.com/tokeninfo";
|
|
432
|
+
const OAUTH_REVOKE_URL = "https://oauth2.googleapis.com/revoke";
|
|
455
433
|
const OAUTH_TIMEOUT_MS = 8e3;
|
|
456
434
|
const OAUTH_MAX_ATTEMPTS = 3;
|
|
457
435
|
const OAUTH_BACKOFF_MS = [
|
|
@@ -460,12 +438,18 @@ const OAUTH_BACKOFF_MS = [
|
|
|
460
438
|
1500
|
|
461
439
|
];
|
|
462
440
|
async function refreshAccessTokenResult(refreshToken, clientId, clientSecret) {
|
|
463
|
-
|
|
441
|
+
const result = await postOAuthTokenResult(new URLSearchParams({
|
|
464
442
|
client_id: clientId,
|
|
465
443
|
client_secret: clientSecret,
|
|
466
444
|
refresh_token: refreshToken,
|
|
467
445
|
grant_type: "refresh_token"
|
|
468
446
|
}), "refresh");
|
|
447
|
+
if (!result.ok) return result;
|
|
448
|
+
return ok({
|
|
449
|
+
accessToken: result.value.accessToken,
|
|
450
|
+
expiresAt: result.value.expiresAt,
|
|
451
|
+
scope: result.value.scope
|
|
452
|
+
});
|
|
469
453
|
}
|
|
470
454
|
async function refreshAccessToken(refreshToken, clientId, clientSecret) {
|
|
471
455
|
return unwrapResult(await refreshAccessTokenResult(refreshToken, clientId, clientSecret), oauthErrorToException);
|
|
@@ -479,19 +463,46 @@ async function exchangeAuthCodeResult(code, clientId, clientSecret, redirectUri)
|
|
|
479
463
|
grant_type: "authorization_code"
|
|
480
464
|
}), "exchange");
|
|
481
465
|
if (!result.ok) return result;
|
|
482
|
-
|
|
466
|
+
return result;
|
|
467
|
+
}
|
|
468
|
+
async function introspectAccessTokenResult(accessToken) {
|
|
469
|
+
const response = await requestOAuthResult(`${OAUTH_TOKEN_INFO_URL}?access_token=${encodeURIComponent(accessToken)}`, { method: "GET" }, "introspect");
|
|
470
|
+
if (!response.ok) return response;
|
|
471
|
+
const parsed = await readOAuthJsonResult(response.value, "introspect");
|
|
472
|
+
if (!parsed.ok) return parsed;
|
|
473
|
+
const data = parsed.value;
|
|
483
474
|
return ok({
|
|
484
|
-
|
|
485
|
-
|
|
475
|
+
issuedTo: optionalString(data.issued_to),
|
|
476
|
+
audience: optionalString(data.aud),
|
|
477
|
+
authorizedParty: optionalString(data.azp),
|
|
478
|
+
userId: optionalString(data.user_id),
|
|
479
|
+
subject: optionalString(data.sub),
|
|
480
|
+
scope: optionalString(data.scope),
|
|
481
|
+
expiresIn: optionalNumber(data.expires_in),
|
|
482
|
+
expiresAt: optionalNumber(data.exp),
|
|
483
|
+
email: optionalString(data.email),
|
|
484
|
+
emailVerified: optionalBoolean(data.email_verified),
|
|
485
|
+
accessType: optionalString(data.access_type)
|
|
486
486
|
});
|
|
487
487
|
}
|
|
488
|
-
async function
|
|
489
|
-
return unwrapResult(await
|
|
488
|
+
async function introspectAccessToken(accessToken) {
|
|
489
|
+
return unwrapResult(await introspectAccessTokenResult(accessToken), oauthErrorToException);
|
|
490
|
+
}
|
|
491
|
+
async function revokeOAuthTokenResult(token) {
|
|
492
|
+
const response = await requestOAuthResult(OAUTH_REVOKE_URL, {
|
|
493
|
+
method: "POST",
|
|
494
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
495
|
+
body: new URLSearchParams({ token })
|
|
496
|
+
}, "revoke");
|
|
497
|
+
return response.ok ? ok(void 0) : response;
|
|
498
|
+
}
|
|
499
|
+
async function revokeOAuthToken(token) {
|
|
500
|
+
return unwrapResult(await revokeOAuthTokenResult(token), oauthErrorToException);
|
|
490
501
|
}
|
|
491
502
|
function oauthHttpError(op, info) {
|
|
492
503
|
const message = `Failed to ${op} token: ${info.message}`;
|
|
493
504
|
const cause = new GscApiError(message, info);
|
|
494
|
-
return info.reason === "invalid_grant" || info.code === 400 || info.code === 401 || info.code === 403 ? {
|
|
505
|
+
return info.reason === "invalid_grant" || info.reason === "invalid_token" || info.code === 400 || info.code === 401 || info.code === 403 ? {
|
|
495
506
|
kind: "auth-expired",
|
|
496
507
|
message,
|
|
497
508
|
cause
|
|
@@ -506,13 +517,28 @@ function oauthErrorToException(error) {
|
|
|
506
517
|
return error.cause instanceof Error ? error.cause : gscErrorToException(error);
|
|
507
518
|
}
|
|
508
519
|
async function postOAuthTokenResult(body, op) {
|
|
520
|
+
const response = await requestOAuthResult(OAUTH_TOKEN_URL, {
|
|
521
|
+
method: "POST",
|
|
522
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
523
|
+
body
|
|
524
|
+
}, op);
|
|
525
|
+
if (!response.ok) return response;
|
|
526
|
+
const parsed = await readOAuthJsonResult(response.value, op);
|
|
527
|
+
if (!parsed.ok) return parsed;
|
|
528
|
+
const data = parsed.value;
|
|
529
|
+
return ok({
|
|
530
|
+
accessToken: data.access_token,
|
|
531
|
+
expiresAt: Math.floor(Date.now() / 1e3) + data.expires_in,
|
|
532
|
+
refreshToken: data.refresh_token,
|
|
533
|
+
scope: data.scope
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
async function requestOAuthResult(url, init, op) {
|
|
509
537
|
let lastError;
|
|
510
538
|
for (let attempt = 0; attempt < OAUTH_MAX_ATTEMPTS; attempt++) {
|
|
511
539
|
if (OAUTH_BACKOFF_MS[attempt]) await new Promise((r) => setTimeout(r, OAUTH_BACKOFF_MS[attempt]));
|
|
512
|
-
const res = await fetch(
|
|
513
|
-
|
|
514
|
-
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
515
|
-
body,
|
|
540
|
+
const res = await fetch(url, {
|
|
541
|
+
...init,
|
|
516
542
|
signal: AbortSignal.timeout(OAUTH_TIMEOUT_MS)
|
|
517
543
|
}).catch((error) => {
|
|
518
544
|
lastError = error;
|
|
@@ -520,12 +546,7 @@ async function postOAuthTokenResult(body, op) {
|
|
|
520
546
|
});
|
|
521
547
|
if (!res) continue;
|
|
522
548
|
if (!res.ok) return err(oauthHttpError(op, parseGoogleError(await res.text(), res.status)));
|
|
523
|
-
|
|
524
|
-
return ok({
|
|
525
|
-
accessToken: data.access_token,
|
|
526
|
-
expiresAt: Math.floor(Date.now() / 1e3) + data.expires_in,
|
|
527
|
-
refresh_token: data.refresh_token
|
|
528
|
-
});
|
|
549
|
+
return ok(res);
|
|
529
550
|
}
|
|
530
551
|
return err({
|
|
531
552
|
kind: "transport",
|
|
@@ -533,6 +554,30 @@ async function postOAuthTokenResult(body, op) {
|
|
|
533
554
|
cause: lastError
|
|
534
555
|
});
|
|
535
556
|
}
|
|
557
|
+
async function readOAuthJsonResult(response, op) {
|
|
558
|
+
try {
|
|
559
|
+
return ok(await response.json());
|
|
560
|
+
} catch (cause) {
|
|
561
|
+
return err({
|
|
562
|
+
kind: "transport",
|
|
563
|
+
message: `Failed to parse ${op} token response`,
|
|
564
|
+
status: response.status,
|
|
565
|
+
cause
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
function optionalString(value) {
|
|
570
|
+
return typeof value === "string" ? value : void 0;
|
|
571
|
+
}
|
|
572
|
+
function optionalNumber(value) {
|
|
573
|
+
const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN;
|
|
574
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
575
|
+
}
|
|
576
|
+
function optionalBoolean(value) {
|
|
577
|
+
if (typeof value === "boolean") return value;
|
|
578
|
+
if (value === "true") return true;
|
|
579
|
+
if (value === "false") return false;
|
|
580
|
+
}
|
|
536
581
|
async function fetchSites(client) {
|
|
537
582
|
return client.sites();
|
|
538
583
|
}
|
|
@@ -668,9 +713,6 @@ function getPstDateDaysAgo(daysAgo) {
|
|
|
668
713
|
pstNow.setDate(pstNow.getDate() - daysAgo);
|
|
669
714
|
return toIsoDate(pstNow);
|
|
670
715
|
}
|
|
671
|
-
function daysAgo(n) {
|
|
672
|
-
return toIsoDate(/* @__PURE__ */ new Date(Date.now() - n * MS_PER_DAY));
|
|
673
|
-
}
|
|
674
716
|
function getLatestGscDate() {
|
|
675
717
|
return getPstDateDaysAgo(3);
|
|
676
718
|
}
|
|
@@ -2328,6 +2370,12 @@ const queryErrors = {
|
|
|
2328
2370
|
message: "Invalid state"
|
|
2329
2371
|
};
|
|
2330
2372
|
},
|
|
2373
|
+
malformedFilterLeaf() {
|
|
2374
|
+
return {
|
|
2375
|
+
kind: "invalid-filter",
|
|
2376
|
+
message: "Malformed filter: each filter leaf requires a string `dimension` and `operator`"
|
|
2377
|
+
};
|
|
2378
|
+
},
|
|
2331
2379
|
unsupportedCapability(capability, context) {
|
|
2332
2380
|
return {
|
|
2333
2381
|
kind: "unsupported-capability",
|
|
@@ -2534,152 +2582,10 @@ function rowWithMetricDefaults(row) {
|
|
|
2534
2582
|
position: row.position ?? 0
|
|
2535
2583
|
};
|
|
2536
2584
|
}
|
|
2537
|
-
function progressBar(current, total, label, width = 30) {
|
|
2538
|
-
const percent = Math.min(current / total, 1);
|
|
2539
|
-
const filled = Math.round(width * percent);
|
|
2540
|
-
const empty = width - filled;
|
|
2541
|
-
return ` ${`\x1B[36m${"█".repeat(filled)}\x1B[0m\x1B[90m${"░".repeat(empty)}\x1B[0m`} \x1B[90m${current}/${total}\x1B[0m ${label}`;
|
|
2542
|
-
}
|
|
2543
|
-
const TRAILING_SLASH_RE = /\/$/;
|
|
2544
|
-
function gscdumpApi(options) {
|
|
2545
|
-
const baseUrl = options.baseUrl?.replace(TRAILING_SLASH_RE, "") || "https://gscdump.com";
|
|
2546
|
-
const fetch = ofetch.create({
|
|
2547
|
-
baseURL: baseUrl,
|
|
2548
|
-
retry: 3,
|
|
2549
|
-
retryDelay: 1e3,
|
|
2550
|
-
retryStatusCodes: [
|
|
2551
|
-
408,
|
|
2552
|
-
425,
|
|
2553
|
-
429,
|
|
2554
|
-
500,
|
|
2555
|
-
502,
|
|
2556
|
-
503,
|
|
2557
|
-
504
|
|
2558
|
-
],
|
|
2559
|
-
headers: {
|
|
2560
|
-
"x-api-key": options.apiKey,
|
|
2561
|
-
"Content-Type": "application/json"
|
|
2562
|
-
}
|
|
2563
|
-
});
|
|
2564
|
-
const rawQuery = async (siteId, body, opts) => {
|
|
2565
|
-
const response = await fetch(`/api/sites/${encodeURIComponent(siteId)}/query`, {
|
|
2566
|
-
method: "POST",
|
|
2567
|
-
body,
|
|
2568
|
-
signal: opts?.signal
|
|
2569
|
-
});
|
|
2570
|
-
return { rows: response.rows.map((row) => {
|
|
2571
|
-
return {
|
|
2572
|
-
keys: response.meta.dimensions.map((dim) => String(row[dim] ?? "")),
|
|
2573
|
-
clicks: row.clicks,
|
|
2574
|
-
impressions: row.impressions,
|
|
2575
|
-
ctr: row.ctr,
|
|
2576
|
-
position: row.position
|
|
2577
|
-
};
|
|
2578
|
-
}) };
|
|
2579
|
-
};
|
|
2580
|
-
return {
|
|
2581
|
-
async *query(siteId, builder, opts) {
|
|
2582
|
-
const state = builder.getState();
|
|
2583
|
-
const body = resolveToBody(state);
|
|
2584
|
-
const totalCap = body.rowLimit;
|
|
2585
|
-
const pageSize = Math.min(totalCap ?? 25e3, 25e3);
|
|
2586
|
-
let startRow = body.startRow || 0;
|
|
2587
|
-
let yielded = 0;
|
|
2588
|
-
let metadata;
|
|
2589
|
-
while (true) {
|
|
2590
|
-
opts?.signal?.throwIfAborted();
|
|
2591
|
-
const remaining = totalCap ? totalCap - yielded : pageSize;
|
|
2592
|
-
if (remaining <= 0) break;
|
|
2593
|
-
const rowLimit = Math.min(pageSize, remaining);
|
|
2594
|
-
const response = await fetch(`/api/sites/${encodeURIComponent(siteId)}/query`, {
|
|
2595
|
-
method: "POST",
|
|
2596
|
-
body: {
|
|
2597
|
-
...body,
|
|
2598
|
-
startRow,
|
|
2599
|
-
rowLimit
|
|
2600
|
-
},
|
|
2601
|
-
signal: opts?.signal
|
|
2602
|
-
});
|
|
2603
|
-
if (response.metadata) metadata = response.metadata;
|
|
2604
|
-
const rows = response.rows.map((row) => {
|
|
2605
|
-
const result = rowWithMetricDefaults(row);
|
|
2606
|
-
state.dimensions.forEach((dim) => {
|
|
2607
|
-
result[dim] = row[dim];
|
|
2608
|
-
});
|
|
2609
|
-
return result;
|
|
2610
|
-
});
|
|
2611
|
-
yield rows;
|
|
2612
|
-
yielded += rows.length;
|
|
2613
|
-
if (!response.meta.hasMore || rows.length < rowLimit) break;
|
|
2614
|
-
startRow += rows.length;
|
|
2615
|
-
}
|
|
2616
|
-
return { metadata };
|
|
2617
|
-
},
|
|
2618
|
-
sites: (() => {
|
|
2619
|
-
const list = async (opts) => {
|
|
2620
|
-
return (await fetch("/api/sites", { signal: opts?.signal })).sites.map((s) => ({
|
|
2621
|
-
siteUrl: s.gscSiteUrl,
|
|
2622
|
-
permissionLevel: s.permissionLevel || "siteOwner"
|
|
2623
|
-
}));
|
|
2624
|
-
};
|
|
2625
|
-
const unsupported = (op) => () => {
|
|
2626
|
-
throw new Error(`sites.${op} not available via gscdump API. Use googleSearchConsole() with OAuth credentials.`);
|
|
2627
|
-
};
|
|
2628
|
-
return Object.assign(list, {
|
|
2629
|
-
list,
|
|
2630
|
-
get: unsupported("get"),
|
|
2631
|
-
add: unsupported("add"),
|
|
2632
|
-
delete: unsupported("delete")
|
|
2633
|
-
});
|
|
2634
|
-
})(),
|
|
2635
|
-
verification: {
|
|
2636
|
-
getToken: () => {
|
|
2637
|
-
throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2638
|
-
},
|
|
2639
|
-
insert: () => {
|
|
2640
|
-
throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2641
|
-
},
|
|
2642
|
-
list: () => {
|
|
2643
|
-
throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2644
|
-
},
|
|
2645
|
-
get: () => {
|
|
2646
|
-
throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2647
|
-
},
|
|
2648
|
-
delete: () => {
|
|
2649
|
-
throw new Error("Site Verification API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2650
|
-
}
|
|
2651
|
-
},
|
|
2652
|
-
inspect: () => {
|
|
2653
|
-
throw new Error("URL inspection not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2654
|
-
},
|
|
2655
|
-
sitemaps: {
|
|
2656
|
-
list: async (siteId, opts) => {
|
|
2657
|
-
return (await fetch(`/api/sites/${encodeURIComponent(siteId)}/sitemaps`, { signal: opts?.signal })).sitemaps || [];
|
|
2658
|
-
},
|
|
2659
|
-
get: () => {
|
|
2660
|
-
throw new Error("Sitemap get not available via gscdump API.");
|
|
2661
|
-
},
|
|
2662
|
-
submit: () => {
|
|
2663
|
-
throw new Error("Sitemap submit not available via gscdump API.");
|
|
2664
|
-
},
|
|
2665
|
-
delete: () => {
|
|
2666
|
-
throw new Error("Sitemap delete not available via gscdump API.");
|
|
2667
|
-
}
|
|
2668
|
-
},
|
|
2669
|
-
indexing: {
|
|
2670
|
-
publish: () => {
|
|
2671
|
-
throw new Error("Indexing API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2672
|
-
},
|
|
2673
|
-
getMetadata: () => {
|
|
2674
|
-
throw new Error("Indexing API not available via gscdump API. Use googleSearchConsole() with OAuth credentials.");
|
|
2675
|
-
}
|
|
2676
|
-
},
|
|
2677
|
-
_rawQuery: rawQuery
|
|
2678
|
-
};
|
|
2679
|
-
}
|
|
2680
2585
|
const GSC_API = "https://searchconsole.googleapis.com";
|
|
2681
2586
|
const INDEXING_API = "https://indexing.googleapis.com";
|
|
2682
2587
|
const SITE_VERIFICATION_API = "https://www.googleapis.com/siteVerification/v1";
|
|
2588
|
+
const DEFAULT_GSC_REQUEST_TIMEOUT_MS = 3e4;
|
|
2683
2589
|
function encodeSiteUrl(siteUrl) {
|
|
2684
2590
|
if (siteUrl.startsWith("sc-domain:")) return `sc-domain:${encodeURIComponent(siteUrl.slice(10))}`;
|
|
2685
2591
|
return encodeURIComponent(siteUrl);
|
|
@@ -2689,6 +2595,7 @@ function assertValidSiteUrl(siteUrl) {
|
|
|
2689
2595
|
if (/^https?:\/\/.+/.test(siteUrl)) return;
|
|
2690
2596
|
throw new Error(`Invalid siteUrl: expected "https?://…" or "sc-domain:…", got "${siteUrl}"`);
|
|
2691
2597
|
}
|
|
2598
|
+
const OAUTH_EXPIRY_SKEW_MS = 6e4;
|
|
2692
2599
|
function createAuth(options) {
|
|
2693
2600
|
let credentials = { refresh_token: options.refreshToken };
|
|
2694
2601
|
let refreshPromise = null;
|
|
@@ -2697,22 +2604,14 @@ function createAuth(options) {
|
|
|
2697
2604
|
return credentials;
|
|
2698
2605
|
},
|
|
2699
2606
|
async getAccessToken() {
|
|
2700
|
-
if (credentials?.access_token && credentials.expiry_date && credentials.expiry_date > Date.now()) return { token: credentials.access_token };
|
|
2701
|
-
refreshPromise ??=
|
|
2702
|
-
method: "POST",
|
|
2703
|
-
body: new URLSearchParams({
|
|
2704
|
-
client_id: options.clientId,
|
|
2705
|
-
client_secret: options.clientSecret,
|
|
2706
|
-
refresh_token: options.refreshToken,
|
|
2707
|
-
grant_type: "refresh_token"
|
|
2708
|
-
})
|
|
2709
|
-
}).then((response) => {
|
|
2607
|
+
if (credentials?.access_token && credentials.expiry_date && credentials.expiry_date > Date.now() + OAUTH_EXPIRY_SKEW_MS) return { token: credentials.access_token };
|
|
2608
|
+
refreshPromise ??= refreshAccessToken(options.refreshToken, options.clientId, options.clientSecret).then((response) => {
|
|
2710
2609
|
credentials = {
|
|
2711
2610
|
...credentials,
|
|
2712
|
-
access_token: response.
|
|
2713
|
-
expiry_date:
|
|
2611
|
+
access_token: response.accessToken,
|
|
2612
|
+
expiry_date: response.expiresAt * 1e3
|
|
2714
2613
|
};
|
|
2715
|
-
return response.
|
|
2614
|
+
return response.accessToken;
|
|
2716
2615
|
}).finally(() => {
|
|
2717
2616
|
refreshPromise = null;
|
|
2718
2617
|
});
|
|
@@ -2734,6 +2633,7 @@ function createFetch(auth, options) {
|
|
|
2734
2633
|
const authState = typeof auth === "object" && auth !== null && "clientId" in auth && "refreshToken" in auth && !("getAccessToken" in auth) ? createAuth(auth) : auth;
|
|
2735
2634
|
return ofetch.create({
|
|
2736
2635
|
...options,
|
|
2636
|
+
timeout: options?.timeout ?? 3e4,
|
|
2737
2637
|
retry: 3,
|
|
2738
2638
|
retryDelay: (ctx) => {
|
|
2739
2639
|
const status = ctx.response?.status;
|
|
@@ -2781,7 +2681,7 @@ function googleSearchConsole(auth, options = {}) {
|
|
|
2781
2681
|
const authState = typeof auth === "object" && auth !== null && "clientId" in auth && "refreshToken" in auth && !("getAccessToken" in auth) ? createAuth(auth) : auth;
|
|
2782
2682
|
if (options.fetch) fetch = options.fetch;
|
|
2783
2683
|
else {
|
|
2784
|
-
const fetchOptions = options.fetchOptions
|
|
2684
|
+
const fetchOptions = { ...options.fetchOptions };
|
|
2785
2685
|
if (options.onRateLimited) {
|
|
2786
2686
|
const originalOnError = fetchOptions.onResponseError;
|
|
2787
2687
|
fetchOptions.onResponseError = async (ctx) => {
|
|
@@ -2792,7 +2692,7 @@ function googleSearchConsole(auth, options = {}) {
|
|
|
2792
2692
|
}
|
|
2793
2693
|
fetch = createFetch(authState, fetchOptions);
|
|
2794
2694
|
}
|
|
2795
|
-
const
|
|
2695
|
+
const querySearchAnalytics = (siteUrl, body, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/searchAnalytics/query`, {
|
|
2796
2696
|
method: "POST",
|
|
2797
2697
|
body,
|
|
2798
2698
|
signal: opts?.signal
|
|
@@ -2812,7 +2712,7 @@ function googleSearchConsole(auth, options = {}) {
|
|
|
2812
2712
|
const remaining = totalCap ? totalCap - yielded : pageSize;
|
|
2813
2713
|
if (remaining <= 0) break;
|
|
2814
2714
|
const rowLimit = Math.min(pageSize, remaining);
|
|
2815
|
-
const response = await
|
|
2715
|
+
const response = await querySearchAnalytics(siteUrl, {
|
|
2816
2716
|
...body,
|
|
2817
2717
|
startRow,
|
|
2818
2718
|
rowLimit
|
|
@@ -2920,7 +2820,7 @@ function googleSearchConsole(auth, options = {}) {
|
|
|
2920
2820
|
signal: opts?.signal
|
|
2921
2821
|
})
|
|
2922
2822
|
},
|
|
2923
|
-
|
|
2823
|
+
searchAnalytics: { query: querySearchAnalytics }
|
|
2924
2824
|
};
|
|
2925
2825
|
}
|
|
2926
2826
|
const INDEXING_ISSUE_FILTERS = {
|
|
@@ -3162,7 +3062,6 @@ function findExactGscProperty(propertyUrl, properties) {
|
|
|
3162
3062
|
function formatGscPropertyCandidates(candidates) {
|
|
3163
3063
|
return candidates.filter((c) => !!c).map((property) => `${property.siteUrl} (${property.permissionLevel})`).join(", ");
|
|
3164
3064
|
}
|
|
3165
|
-
const URL_INSPECTION_DAILY_LIMIT = 2e3;
|
|
3166
3065
|
const URL_INSPECTION_EFFECTIVE_LIMIT = 1800;
|
|
3167
3066
|
const URL_PROTOCOL_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
3168
3067
|
function normalizeUrl(input) {
|
|
@@ -3270,4 +3169,4 @@ function urlMatchKey(url) {
|
|
|
3270
3169
|
if (!u) return null;
|
|
3271
3170
|
return [u.hostname.replace(/^www\./, ""), u.pathname.replace(/\/$/, "") || "/"].join("").toLowerCase();
|
|
3272
3171
|
}
|
|
3273
|
-
export { DAYS_PER_RANGE, GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_INDEXING_SCOPE,
|
|
3172
|
+
export { DAYS_PER_RANGE, DEFAULT_GSC_REQUEST_TIMEOUT_MS, GSCDUMP_ONBOARDING_CONTRACT_VERSION, GSCDUMP_OPTIONAL_INDEXING_SCOPE, GSCDUMP_REQUIRED_ANALYTICS_SCOPE, GSCDUMP_WRITE_ANALYTICS_SCOPE, GSC_FINALIZED_LAG_DAYS, GSC_FRESHEST_LAG_DAYS, GSC_INDEXING_SCOPE, GSC_READ_SCOPE, GSC_RETENTION_MONTHS, GSC_SITE_VERIFICATION_SCOPE, GSC_WRITE_SCOPE, GscApiError, INDEXING_ISSUE_FILTERS, INDEXING_ISSUE_LABELS, INDEXING_ISSUE_SEVERITY, KNOWN_COVERAGE_STATES, KNOWN_PAGE_FETCH_STATES, MAX_FLAT_INSPECTION_BATCH_CONCURRENCY, MS_PER_DAY, URL_INSPECTION_EFFECTIVE_LIMIT, accountNextActions, accountStatuses, addDays, addSite, analyticsNextActions, analyticsStatuses, batchInspectUrls, batchInspectUrlsFlatSettled, batchRequestIndexing, canUseUrlInspection, classifyError, countDays, createAuth, createFetch, decodeSiteId, deleteSite, deleteSitemap, discoverSitemap, encodeSiteId, err, exchangeAuthCodeResult, fetchSitemap, fetchSitemapUrls, fetchSitemaps, fetchSites, fetchSitesWithSitemaps, findBestGscProperty, findExactGscProperty, formatGscPropertyCandidates, generateGscDateRange, getBackfillProgress, getDateRange, getFreshestGscDate, getIndexingEligibility, getIndexingMetadata, getLatestGscDate, getNextCheckAfter, getNextCheckPriority, getNextDate, getOldestGscDate, getPendingDates, getPreviousDate, getPstDate, getVerificationToken, getVerifiedSite, googleSearchConsole, groupIntoRanges, gscErrorToException, gscPropertyMatchesTarget, hasGoogleScope, hasGscReadScope, hasGscWriteScope, hasIndexingScope, hasOptionalIndexingScope, hasRequiredAnalyticsScope, indexingNextActions, indexingStatuses, inspectUrl, inspectUrlFlat, introspectAccessToken, introspectAccessTokenResult, isErr, isOk, isPermissionDeniedError, isValidGscDate, isVerifiedGscPermission, isVerifiedGscProperty, lifecycleErrorCodes, listVerifiedSites, matchGscSite, normalizeGscSiteUrl, normalizeRegistrationTarget, normalizeSiteUrl, normalizeUrl, ok, parseGoogleError, parseGrantedScopes, parseGscSiteUrl, pickBestGscProperty, propertyNextActions, propertyStatuses, querySourceModes, refreshAccessToken, requestIndexing, resolveVerificationTarget, rethrowAsGscApiError, revokeOAuthToken, revokeOAuthTokenResult, runSequentialBatch, siteUrlToVerificationSite, sitemapNextActions, sitemapStatuses, submitSitemap, toIsoDate, unmappedInspectionReasons, unverifySite, unwrapResult, urlMatchKey, verificationMethodsFor, verifySite };
|
package/dist/query/index.d.mts
CHANGED
|
@@ -169,7 +169,7 @@ declare const impressions: MetricColumn<"impressions">;
|
|
|
169
169
|
declare const ctr: MetricColumn<"ctr">;
|
|
170
170
|
declare const position: MetricColumn<"position">;
|
|
171
171
|
declare const searchType: QueryParam<"searchType">;
|
|
172
|
-
type QueryErrorKind = 'missing-date-range' | 'invalid-row-limit' | 'invalid-start-row' | 'invalid-data-state' | 'invalid-aggregation-type' | 'invalid-builder-state' | 'unsupported-capability' | 'unresolvable-dataset';
|
|
172
|
+
type QueryErrorKind = 'missing-date-range' | 'invalid-row-limit' | 'invalid-start-row' | 'invalid-data-state' | 'invalid-aggregation-type' | 'invalid-builder-state' | 'invalid-filter' | 'unsupported-capability' | 'unresolvable-dataset';
|
|
173
173
|
type QueryError = {
|
|
174
174
|
kind: 'missing-date-range';
|
|
175
175
|
message: string;
|
|
@@ -191,6 +191,9 @@ type QueryError = {
|
|
|
191
191
|
kind: 'invalid-builder-state';
|
|
192
192
|
message: string;
|
|
193
193
|
cause?: unknown;
|
|
194
|
+
} | {
|
|
195
|
+
kind: 'invalid-filter';
|
|
196
|
+
message: string;
|
|
194
197
|
} | {
|
|
195
198
|
kind: 'unsupported-capability';
|
|
196
199
|
capability: string;
|
|
@@ -214,12 +217,11 @@ declare const queryErrors: {
|
|
|
214
217
|
readonly byNewsShowcaseNotAllowedWithPage: () => QueryError;
|
|
215
218
|
readonly byNewsShowcaseRequiresShowcaseFilter: () => QueryError;
|
|
216
219
|
readonly invalidBuilderState: (cause?: unknown) => QueryError;
|
|
220
|
+
readonly malformedFilterLeaf: () => QueryError;
|
|
217
221
|
readonly unsupportedCapability: (capability: string, context: string) => QueryError;
|
|
218
222
|
readonly unresolvableDataset: (dimensions: readonly Dimension[], filterDims?: readonly Dimension[]) => QueryError;
|
|
219
223
|
};
|
|
220
224
|
declare function isQueryError(value: unknown): value is QueryError;
|
|
221
|
-
/** The human-readable rendering of a `QueryError`, for logs and string sinks. */
|
|
222
|
-
declare function formatQueryError(error: QueryError): string;
|
|
223
225
|
/**
|
|
224
226
|
* Thrown when a query needs a planner capability (regex pushdown, comparison
|
|
225
227
|
* joins, multi-dataset reads) the target engine lacks. Engines catch it to fall
|
|
@@ -353,14 +355,15 @@ declare function buildLogicalPlan(state: BuilderState, capabilities?: PlannerCap
|
|
|
353
355
|
*/
|
|
354
356
|
declare function buildLogicalComparisonPlanResult(current: BuilderState, previous: BuilderState, capabilities?: PlannerCapabilities, comparisonFilter?: ComparisonFilter): Result<LogicalComparisonPlan, QueryError>;
|
|
355
357
|
declare function buildLogicalComparisonPlan(current: BuilderState, previous: BuilderState, capabilities?: PlannerCapabilities, comparisonFilter?: ComparisonFilter): LogicalComparisonPlan;
|
|
356
|
-
declare function isJsonFilter(value: unknown): value is JsonFilter;
|
|
357
|
-
declare function parseJsonFilter(json: JsonFilter): Filter<any>;
|
|
358
358
|
declare function normalizeFilter(input?: FilterInput): Filter<any> | undefined;
|
|
359
359
|
/**
|
|
360
360
|
* Errors-as-values core for {@link normalizeBuilderState}: returns an
|
|
361
361
|
* `invalid-builder-state` `QueryError` when the untrusted partner-API body is
|
|
362
|
-
* not an object,
|
|
363
|
-
*
|
|
362
|
+
* not an object, and an `invalid-filter` `QueryError` when a filter leaf lacks
|
|
363
|
+
* its string `dimension`/`operator`, instead of throwing. Also coerces
|
|
364
|
+
* alternative `orderBy` shapes into the canonical `{ column, dir }`.
|
|
365
|
+
* Receive-edge parse (parse, don't validate), so hosts can map a bad body to a
|
|
366
|
+
* 4xx and downstream consumers only ever see the canonical shape.
|
|
364
367
|
*/
|
|
365
368
|
declare function normalizeBuilderStateResult(state: unknown): Result<BuilderState, QueryError>;
|
|
366
369
|
declare function normalizeBuilderState(state: unknown): BuilderState;
|
|
@@ -389,4 +392,4 @@ declare function resolveToBody(state: BuilderState): GscSearchAnalyticsRequest;
|
|
|
389
392
|
declare function currentPstDate(): string;
|
|
390
393
|
declare function today(): string;
|
|
391
394
|
declare function daysAgo(n: number): string;
|
|
392
|
-
export { type BuilderState, type Column, type ComparisonFilter, Countries, type Country, type Device, Devices, type Dimension, type DimensionValueMap, type Filter, type FilterInput, type GSCQueryBuilder, type GSCResult, type GSCRow, type InternalFilter, type JsonFilter, type JsonInternalFilter, type LogicalComparisonPlan, type LogicalDataset, type LogicalDimensionFilter, type LogicalMetricFilter, type LogicalQueryPlan, type Metric, type MetricColumn, type PlannerCapabilities, QueryError, QueryErrorKind, type QueryParam, type QueryParamName, type QueryParamValueMap, type SearchType, SearchTypes, UnresolvableDatasetError, UnsupportedLogicalCapabilityError, and, between, buildLogicalComparisonPlan, buildLogicalComparisonPlanResult, buildLogicalPlan, buildLogicalPlanResult, clicks, contains, country, ctr, currentPstDate, date, daysAgo, device, eq, extractDateRange, extractMetricFilters, extractSearchType, extractSpecialOperatorFilters,
|
|
395
|
+
export { type BuilderState, type Column, type ComparisonFilter, Countries, type Country, type Device, Devices, type Dimension, type DimensionValueMap, type Filter, type FilterInput, type GSCQueryBuilder, type GSCResult, type GSCRow, type InternalFilter, type JsonFilter, type JsonInternalFilter, type LogicalComparisonPlan, type LogicalDataset, type LogicalDimensionFilter, type LogicalMetricFilter, type LogicalQueryPlan, type Metric, type MetricColumn, type PlannerCapabilities, QueryError, QueryErrorKind, type QueryParam, type QueryParamName, type QueryParamValueMap, type SearchType, SearchTypes, UnresolvableDatasetError, UnsupportedLogicalCapabilityError, and, between, buildLogicalComparisonPlan, buildLogicalComparisonPlanResult, buildLogicalPlan, buildLogicalPlanResult, clicks, contains, country, ctr, currentPstDate, date, daysAgo, device, eq, extractDateRange, extractMetricFilters, extractSearchType, extractSpecialOperatorFilters, gsc, gt, gte, hour, impressions, inArray, isQueryError, like, lt, lte, ne, normalizeBuilderState, normalizeBuilderStateResult, normalizeFilter, not, notRegex, or, page, position, query, queryCanonical, queryErrorToException, queryErrors, regex, resolveToBody, resolveToBodyResult, searchAppearance, searchType, today, topLevel };
|