gscdump 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/batch.d.mts +12 -0
- package/dist/api/batch.mjs +29 -0
- package/dist/api/indexing.d.mts +37 -0
- package/dist/api/indexing.mjs +25 -0
- package/dist/api/inspection.d.mts +89 -0
- package/dist/api/inspection.mjs +128 -0
- package/dist/api/oauth.d.mts +48 -0
- package/dist/api/oauth.mjs +154 -0
- package/dist/api/sites.d.mts +45 -0
- package/dist/api/sites.mjs +34 -0
- package/dist/api/verification.d.mts +43 -0
- package/dist/api/verification.mjs +89 -0
- package/dist/contracts.d.mts +46 -1
- package/dist/core/cli-format.mjs +9 -0
- package/dist/core/client.d.mts +122 -0
- package/dist/core/client.mjs +246 -0
- package/dist/core/errors.d.mts +66 -0
- package/dist/core/errors.mjs +207 -0
- package/dist/core/indexing-issues.d.mts +60 -0
- package/dist/core/indexing-issues.mjs +139 -0
- package/dist/core/property.d.mts +41 -0
- package/dist/core/property.mjs +74 -0
- package/dist/core/quota.d.mts +2 -0
- package/dist/core/quota.mjs +2 -0
- package/dist/core/result.d.mts +19 -1
- package/dist/core/scope-values.d.mts +7 -0
- package/dist/core/scope-values.mjs +15 -0
- package/dist/core/scopes.d.mts +5 -0
- package/dist/core/scopes.mjs +11 -0
- package/dist/core/site-url.d.mts +25 -0
- package/dist/core/site-url.mjs +30 -0
- package/dist/core/types.d.mts +201 -0
- package/dist/core/types.mjs +1 -0
- package/dist/dates.d.mts +2 -2
- package/dist/dates.mjs +2 -2
- package/dist/index.d.mts +20 -814
- package/dist/index.mjs +19 -1243
- package/dist/onboarding.d.mts +2 -0
- package/dist/onboarding.mjs +2 -0
- package/dist/{_chunks → query}/builder.d.mts +2 -2
- package/dist/query/builder.mjs +86 -0
- package/dist/query/columns.d.mts +15 -0
- package/dist/query/columns.mjs +23 -0
- package/dist/query/constants.d.mts +19 -0
- package/dist/query/constants.mjs +16 -0
- package/dist/query/errors.d.mts +85 -0
- package/dist/query/errors.mjs +141 -0
- package/dist/query/index.d.mts +9 -75
- package/dist/query/index.mjs +7 -230
- package/dist/query/operator-meta.mjs +41 -0
- package/dist/query/operators.d.mts +24 -0
- package/dist/query/operators.mjs +124 -0
- package/dist/query/plan.d.mts +92 -2
- package/dist/query/plan.mjs +3 -1
- package/dist/query/resolver.d.mts +40 -0
- package/dist/query/resolver.mjs +243 -0
- package/dist/{_chunks → query}/types.d.mts +3 -25
- package/dist/query/utils/countries.d.mts +7 -0
- package/dist/{_chunks/resolver.mjs → query/utils/countries.mjs} +1 -434
- package/dist/{_chunks → query/utils}/dayjs.mjs +1 -1
- package/dist/sitemap.d.mts +26 -0
- package/dist/sitemap.mjs +70 -0
- package/dist/url.d.mts +9 -0
- package/dist/url.mjs +6 -0
- package/package.json +2 -2
- package/dist/_chunks/contracts.d.mts +0 -47
- package/dist/_chunks/plan.d.mts +0 -175
- package/dist/_chunks/result.d.mts +0 -20
- /package/dist/{_chunks → core}/gsc-dates.d.mts +0 -0
- /package/dist/{_chunks → core}/gsc-dates.mjs +0 -0
- /package/dist/{_chunks → query/utils}/dayjs.d.mts +0 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
const SC_DOMAIN_PREFIX = "sc-domain:";
|
|
2
|
+
function bareDomain(siteUrl) {
|
|
3
|
+
if (siteUrl.startsWith(SC_DOMAIN_PREFIX)) return siteUrl.slice(10);
|
|
4
|
+
try {
|
|
5
|
+
return new URL(siteUrl.startsWith("http") ? siteUrl : `https://${siteUrl}`).host;
|
|
6
|
+
} catch {
|
|
7
|
+
return siteUrl;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
function urlPrefix(siteUrl) {
|
|
11
|
+
try {
|
|
12
|
+
const url = new URL(siteUrl.startsWith("http") ? siteUrl : `https://${siteUrl}`);
|
|
13
|
+
return `${url.protocol}//${url.host}/`;
|
|
14
|
+
} catch {
|
|
15
|
+
return siteUrl;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function siteUrlToVerificationSite(siteUrl) {
|
|
19
|
+
if (siteUrl.startsWith(SC_DOMAIN_PREFIX)) return {
|
|
20
|
+
type: "INET_DOMAIN",
|
|
21
|
+
identifier: siteUrl.slice(10)
|
|
22
|
+
};
|
|
23
|
+
return {
|
|
24
|
+
type: "SITE",
|
|
25
|
+
identifier: siteUrl
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function resolveVerificationTarget(siteUrl, method) {
|
|
29
|
+
const isDomainProperty = siteUrl.startsWith(SC_DOMAIN_PREFIX);
|
|
30
|
+
const chosen = method ?? (isDomainProperty ? "DNS_TXT" : "META");
|
|
31
|
+
const isDns = chosen === "DNS_TXT" || chosen === "DNS_CNAME";
|
|
32
|
+
if (isDomainProperty && !isDns) return {
|
|
33
|
+
site: {
|
|
34
|
+
type: "INET_DOMAIN",
|
|
35
|
+
identifier: bareDomain(siteUrl)
|
|
36
|
+
},
|
|
37
|
+
method: "DNS_TXT"
|
|
38
|
+
};
|
|
39
|
+
if (isDns) return {
|
|
40
|
+
site: {
|
|
41
|
+
type: "INET_DOMAIN",
|
|
42
|
+
identifier: bareDomain(siteUrl)
|
|
43
|
+
},
|
|
44
|
+
method: chosen
|
|
45
|
+
};
|
|
46
|
+
return {
|
|
47
|
+
site: {
|
|
48
|
+
type: "SITE",
|
|
49
|
+
identifier: urlPrefix(siteUrl)
|
|
50
|
+
},
|
|
51
|
+
method: chosen
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function verificationMethodsFor(site) {
|
|
55
|
+
if (site.type === "INET_DOMAIN") return ["DNS_TXT", "DNS_CNAME"];
|
|
56
|
+
return [
|
|
57
|
+
"META",
|
|
58
|
+
"FILE",
|
|
59
|
+
"ANALYTICS",
|
|
60
|
+
"TAG_MANAGER"
|
|
61
|
+
];
|
|
62
|
+
}
|
|
63
|
+
async function getVerificationToken(client, siteUrl, method) {
|
|
64
|
+
const { site, method: verificationMethod } = resolveVerificationTarget(siteUrl, method);
|
|
65
|
+
return {
|
|
66
|
+
...await client.verification.getToken({
|
|
67
|
+
site,
|
|
68
|
+
verificationMethod
|
|
69
|
+
}),
|
|
70
|
+
site
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
async function verifySite(client, siteUrl, method) {
|
|
74
|
+
const { site, method: verificationMethod } = resolveVerificationTarget(siteUrl, method);
|
|
75
|
+
return client.verification.insert({
|
|
76
|
+
site,
|
|
77
|
+
verificationMethod
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
async function listVerifiedSites(client) {
|
|
81
|
+
return client.verification.list();
|
|
82
|
+
}
|
|
83
|
+
async function getVerifiedSite(client, id) {
|
|
84
|
+
return client.verification.get(id);
|
|
85
|
+
}
|
|
86
|
+
async function unverifySite(client, id) {
|
|
87
|
+
return client.verification.delete(id);
|
|
88
|
+
}
|
|
89
|
+
export { getVerificationToken, getVerifiedSite, listVerifiedSites, resolveVerificationTarget, siteUrlToVerificationSite, unverifySite, verificationMethodsFor, verifySite };
|
package/dist/contracts.d.mts
CHANGED
|
@@ -1,2 +1,47 @@
|
|
|
1
|
-
import { ColumnDef, ColumnType,
|
|
1
|
+
import { ColumnDef, ColumnType, Row, TableName, TableSchema, TenantCtx } from "@gscdump/contracts";
|
|
2
|
+
type GscSearchAnalyticsDimension = 'page' | 'query' | 'country' | 'device' | 'date' | 'hour' | 'searchAppearance';
|
|
3
|
+
type GscDataState = 'final' | 'all' | 'hourly_all';
|
|
4
|
+
interface GscSearchAnalyticsMetadata {
|
|
5
|
+
/** First date (YYYY-MM-DD, PT) still being collected. Populated when dataState=`all` and grouped by `date`. */
|
|
6
|
+
first_incomplete_date?: string;
|
|
7
|
+
/** First hour (YYYY-MM-DDThh:mm:ss±hh:mm, PT) still being collected. Populated when dataState=`hourly_all` and grouped by `hour`. */
|
|
8
|
+
first_incomplete_hour?: string;
|
|
9
|
+
}
|
|
10
|
+
type GscSearchAnalyticsFilterOperator = 'equals' | 'notEquals' | 'contains' | 'notContains' | 'includingRegex' | 'excludingRegex';
|
|
11
|
+
interface GscSearchAnalyticsFilter {
|
|
12
|
+
dimension: GscSearchAnalyticsDimension;
|
|
13
|
+
expression: string;
|
|
14
|
+
operator?: GscSearchAnalyticsFilterOperator;
|
|
15
|
+
}
|
|
16
|
+
interface GscSearchAnalyticsFilterGroup {
|
|
17
|
+
groupType?: 'and' | 'or';
|
|
18
|
+
filters: GscSearchAnalyticsFilter[];
|
|
19
|
+
}
|
|
20
|
+
type GscSearchType = 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews';
|
|
21
|
+
type GscAggregationType = 'auto' | 'byPage' | 'byProperty' | 'byNewsShowcasePanel';
|
|
22
|
+
type GscResponseAggregationType = 'auto' | 'byPage' | 'byProperty' | 'byNewsShowcasePanel';
|
|
23
|
+
interface GscSearchAnalyticsRequest {
|
|
24
|
+
startDate: string;
|
|
25
|
+
endDate: string;
|
|
26
|
+
dimensions?: GscSearchAnalyticsDimension[];
|
|
27
|
+
dimensionFilterGroups?: GscSearchAnalyticsFilterGroup[];
|
|
28
|
+
rowLimit?: number;
|
|
29
|
+
startRow?: number;
|
|
30
|
+
/** GSC search corpus. Maps to wire field `type` (the API still accepts the deprecated `searchType` alias). */
|
|
31
|
+
type?: GscSearchType;
|
|
32
|
+
dataState?: GscDataState;
|
|
33
|
+
aggregationType?: GscAggregationType;
|
|
34
|
+
}
|
|
35
|
+
interface GscSearchAnalyticsRow {
|
|
36
|
+
keys: string[];
|
|
37
|
+
clicks: number;
|
|
38
|
+
impressions: number;
|
|
39
|
+
ctr: number;
|
|
40
|
+
position: number;
|
|
41
|
+
}
|
|
42
|
+
interface GscSearchAnalyticsResponse {
|
|
43
|
+
rows?: GscSearchAnalyticsRow[];
|
|
44
|
+
responseAggregationType?: GscResponseAggregationType;
|
|
45
|
+
metadata?: GscSearchAnalyticsMetadata;
|
|
46
|
+
}
|
|
2
47
|
export { type ColumnDef, type ColumnType, GscAggregationType, GscDataState, GscResponseAggregationType, GscSearchAnalyticsDimension, GscSearchAnalyticsFilter, GscSearchAnalyticsFilterGroup, GscSearchAnalyticsFilterOperator, GscSearchAnalyticsMetadata, GscSearchAnalyticsRequest, GscSearchAnalyticsResponse, GscSearchAnalyticsRow, GscSearchType, type Row, type TableName, type TableSchema, type TenantCtx };
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { GscResponseAggregationType, GscSearchAnalyticsMetadata } from "../contracts.mjs";
|
|
2
|
+
import { Dimension, GSCRow } from "../query/types.mjs";
|
|
3
|
+
import { GSCQueryBuilder } from "../query/builder.mjs";
|
|
4
|
+
import { ApiSite, ApiSitemap, InspectUrlIndexResponse, PublishUrlNotificationResponse, SearchAnalyticsQuery, SearchAnalyticsResponse, UrlNotificationMetadata } from "./types.mjs";
|
|
5
|
+
import { $Fetch, FetchOptions } from "ofetch";
|
|
6
|
+
/** Default deadline for each direct Google API request. */
|
|
7
|
+
declare const DEFAULT_GSC_REQUEST_TIMEOUT_MS = 30000;
|
|
8
|
+
type VerificationMethod = 'META' | 'FILE' | 'DNS_TXT' | 'DNS_CNAME' | 'ANALYTICS' | 'TAG_MANAGER';
|
|
9
|
+
type VerificationSiteType = 'SITE' | 'INET_DOMAIN' | 'ANDROID_APP';
|
|
10
|
+
interface VerificationSite {
|
|
11
|
+
type: VerificationSiteType;
|
|
12
|
+
identifier: string;
|
|
13
|
+
}
|
|
14
|
+
interface VerificationToken {
|
|
15
|
+
method: string;
|
|
16
|
+
token: string;
|
|
17
|
+
}
|
|
18
|
+
interface VerificationWebResource {
|
|
19
|
+
id?: string;
|
|
20
|
+
site: VerificationSite;
|
|
21
|
+
owners?: string[];
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Compatible interface with OAuth2Client from google-auth-library
|
|
25
|
+
*/
|
|
26
|
+
interface AuthClient {
|
|
27
|
+
credentials?: {
|
|
28
|
+
access_token?: string | null;
|
|
29
|
+
refresh_token?: string | null;
|
|
30
|
+
expiry_date?: number | null;
|
|
31
|
+
};
|
|
32
|
+
getAccessToken: () => Promise<{
|
|
33
|
+
token?: string | null;
|
|
34
|
+
}>;
|
|
35
|
+
}
|
|
36
|
+
interface AuthOptions {
|
|
37
|
+
clientId: string;
|
|
38
|
+
clientSecret: string;
|
|
39
|
+
refreshToken: string;
|
|
40
|
+
}
|
|
41
|
+
declare function createAuth(options: AuthOptions): AuthClient;
|
|
42
|
+
/** Auth can be a token string, object with accessToken, or OAuth2Client-like object with credentials */
|
|
43
|
+
type Auth = string | {
|
|
44
|
+
accessToken: string;
|
|
45
|
+
} | AuthClient | AuthOptions;
|
|
46
|
+
declare function createFetch(auth: Auth, options?: FetchOptions): $Fetch;
|
|
47
|
+
/** Per-call options. `signal` cancels the in-flight request (and, for `query`, the next page too). */
|
|
48
|
+
interface CallOptions {
|
|
49
|
+
signal?: AbortSignal;
|
|
50
|
+
}
|
|
51
|
+
/** Generator return value for `client.query` — exposes API response metadata plus the resolved aggregation type Google actually used (may differ from the requested `aggregationType: 'auto'`). */
|
|
52
|
+
interface QueryReturn {
|
|
53
|
+
metadata?: GscSearchAnalyticsMetadata;
|
|
54
|
+
/** Aggregation type Google actually used. Useful when requesting `auto` to know if rows were aggregated `byPage` vs `byProperty`. */
|
|
55
|
+
responseAggregationType?: GscResponseAggregationType;
|
|
56
|
+
}
|
|
57
|
+
interface GoogleSearchConsoleClient {
|
|
58
|
+
/** Query search analytics with builder, returns async generator yielding typed row batches */
|
|
59
|
+
query: <D extends Dimension[], C>(siteUrl: string, builder: GSCQueryBuilder<D, C>, opts?: CallOptions) => AsyncGenerator<GSCRow<D, C>[], QueryReturn>;
|
|
60
|
+
/**
|
|
61
|
+
* List all sites. Also exposes write ops as `client.sites.add(siteUrl)` and
|
|
62
|
+
* `client.sites.delete(siteUrl)`. Calling `client.sites()` is equivalent to
|
|
63
|
+
* `client.sites.list()`.
|
|
64
|
+
*/
|
|
65
|
+
sites: ((opts?: CallOptions) => Promise<ApiSite[]>) & {
|
|
66
|
+
list: (opts?: CallOptions) => Promise<ApiSite[]>;
|
|
67
|
+
/** Retrieve a single property (with permission level). 404 if not in the user's account. */
|
|
68
|
+
get: (siteUrl: string, opts?: CallOptions) => Promise<ApiSite>;
|
|
69
|
+
/** Add a property in unverified state. Caller must verify ownership separately. */
|
|
70
|
+
add: (siteUrl: string, opts?: CallOptions) => Promise<void>;
|
|
71
|
+
/** Remove a property from the user's account. */
|
|
72
|
+
delete: (siteUrl: string, opts?: CallOptions) => Promise<void>;
|
|
73
|
+
};
|
|
74
|
+
/** Site Verification API (siteverification.googleapis.com). Required to flip a property from unverified to verified. */
|
|
75
|
+
verification: {
|
|
76
|
+
/** Returns the token to place on the site/DNS, plus the resolved method. */
|
|
77
|
+
getToken: (params: {
|
|
78
|
+
site: VerificationSite;
|
|
79
|
+
verificationMethod: VerificationMethod;
|
|
80
|
+
}, opts?: CallOptions) => Promise<VerificationToken>;
|
|
81
|
+
/** Triggers Google to fetch + validate; returns the verified WebResource. */
|
|
82
|
+
insert: (params: {
|
|
83
|
+
site: VerificationSite;
|
|
84
|
+
verificationMethod: VerificationMethod;
|
|
85
|
+
}, opts?: CallOptions) => Promise<VerificationWebResource>;
|
|
86
|
+
list: (opts?: CallOptions) => Promise<VerificationWebResource[]>;
|
|
87
|
+
get: (id: string, opts?: CallOptions) => Promise<VerificationWebResource>;
|
|
88
|
+
delete: (id: string, opts?: CallOptions) => Promise<void>;
|
|
89
|
+
};
|
|
90
|
+
/** Inspect a URL. `languageCode` is a BCP-47 tag for translating result strings; omit to let Google pick. */
|
|
91
|
+
inspect: (siteUrl: string, url: string, opts?: CallOptions & {
|
|
92
|
+
languageCode?: string;
|
|
93
|
+
}) => Promise<InspectUrlIndexResponse>;
|
|
94
|
+
/** Sitemap operations */
|
|
95
|
+
sitemaps: {
|
|
96
|
+
/** List sitemaps. Pass `sitemapIndex` to list children of a sitemap-index file. */
|
|
97
|
+
list: (siteUrl: string, opts?: CallOptions & {
|
|
98
|
+
sitemapIndex?: string;
|
|
99
|
+
}) => Promise<ApiSitemap[]>;
|
|
100
|
+
get: (siteUrl: string, feedpath: string, opts?: CallOptions) => Promise<ApiSitemap>;
|
|
101
|
+
submit: (siteUrl: string, feedpath: string, opts?: CallOptions) => Promise<void>;
|
|
102
|
+
delete: (siteUrl: string, feedpath: string, opts?: CallOptions) => Promise<void>;
|
|
103
|
+
};
|
|
104
|
+
/** Indexing API operations */
|
|
105
|
+
indexing: {
|
|
106
|
+
publish: (url: string, type: 'URL_UPDATED' | 'URL_DELETED', opts?: CallOptions) => Promise<PublishUrlNotificationResponse>;
|
|
107
|
+
getMetadata: (url: string, opts?: CallOptions) => Promise<UrlNotificationMetadata>;
|
|
108
|
+
};
|
|
109
|
+
/** Direct Search Analytics API operations for callers that already have a request body. */
|
|
110
|
+
searchAnalytics: {
|
|
111
|
+
query: (siteUrl: string, body: SearchAnalyticsQuery, opts?: CallOptions) => Promise<SearchAnalyticsResponse>;
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
interface GoogleSearchConsoleClientOptions {
|
|
115
|
+
fetchOptions?: FetchOptions;
|
|
116
|
+
fetch?: $Fetch;
|
|
117
|
+
onRateLimited?: (context: {
|
|
118
|
+
response: Response;
|
|
119
|
+
}) => void | Promise<void>;
|
|
120
|
+
}
|
|
121
|
+
declare function googleSearchConsole(auth: Auth, options?: GoogleSearchConsoleClientOptions): GoogleSearchConsoleClient;
|
|
122
|
+
export { Auth, AuthClient, AuthOptions, CallOptions, DEFAULT_GSC_REQUEST_TIMEOUT_MS, GoogleSearchConsoleClient, GoogleSearchConsoleClientOptions, QueryReturn, VerificationMethod, VerificationSite, VerificationSiteType, VerificationToken, VerificationWebResource, createAuth, createFetch, googleSearchConsole };
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { refreshAccessToken } from "../api/oauth.mjs";
|
|
2
|
+
import { resolveToBody } from "../query/resolver.mjs";
|
|
3
|
+
import { rowWithMetricDefaults } from "./cli-format.mjs";
|
|
4
|
+
import { ofetch } from "ofetch";
|
|
5
|
+
const GSC_API = "https://searchconsole.googleapis.com";
|
|
6
|
+
const INDEXING_API = "https://indexing.googleapis.com";
|
|
7
|
+
const SITE_VERIFICATION_API = "https://www.googleapis.com/siteVerification/v1";
|
|
8
|
+
const DEFAULT_GSC_REQUEST_TIMEOUT_MS = 3e4;
|
|
9
|
+
function encodeSiteUrl(siteUrl) {
|
|
10
|
+
if (siteUrl.startsWith("sc-domain:")) return `sc-domain:${encodeURIComponent(siteUrl.slice(10))}`;
|
|
11
|
+
return encodeURIComponent(siteUrl);
|
|
12
|
+
}
|
|
13
|
+
function assertValidSiteUrl(siteUrl) {
|
|
14
|
+
if (siteUrl.startsWith("sc-domain:") && siteUrl.length > 10) return;
|
|
15
|
+
if (/^https?:\/\/.+/.test(siteUrl)) return;
|
|
16
|
+
throw new Error(`Invalid siteUrl: expected "https?://…" or "sc-domain:…", got "${siteUrl}"`);
|
|
17
|
+
}
|
|
18
|
+
const OAUTH_EXPIRY_SKEW_MS = 6e4;
|
|
19
|
+
function createAuth(options) {
|
|
20
|
+
let credentials = { refresh_token: options.refreshToken };
|
|
21
|
+
let refreshPromise = null;
|
|
22
|
+
return {
|
|
23
|
+
get credentials() {
|
|
24
|
+
return credentials;
|
|
25
|
+
},
|
|
26
|
+
async getAccessToken() {
|
|
27
|
+
if (credentials?.access_token && credentials.expiry_date && credentials.expiry_date > Date.now() + OAUTH_EXPIRY_SKEW_MS) return { token: credentials.access_token };
|
|
28
|
+
refreshPromise ??= refreshAccessToken(options.refreshToken, options.clientId, options.clientSecret).then((response) => {
|
|
29
|
+
credentials = {
|
|
30
|
+
...credentials,
|
|
31
|
+
access_token: response.accessToken,
|
|
32
|
+
expiry_date: response.expiresAt * 1e3
|
|
33
|
+
};
|
|
34
|
+
return response.accessToken;
|
|
35
|
+
}).finally(() => {
|
|
36
|
+
refreshPromise = null;
|
|
37
|
+
});
|
|
38
|
+
return { token: await refreshPromise };
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
async function resolveToken(auth) {
|
|
43
|
+
if (typeof auth === "string") return auth;
|
|
44
|
+
if ("accessToken" in auth && typeof auth.accessToken === "string") return auth.accessToken;
|
|
45
|
+
if ("getAccessToken" in auth && typeof auth.getAccessToken === "function") {
|
|
46
|
+
const { token } = await auth.getAccessToken();
|
|
47
|
+
return token || "";
|
|
48
|
+
}
|
|
49
|
+
if ("credentials" in auth && auth.credentials) return auth.credentials.access_token || "";
|
|
50
|
+
return "";
|
|
51
|
+
}
|
|
52
|
+
function createFetch(auth, options) {
|
|
53
|
+
const authState = typeof auth === "object" && auth !== null && "clientId" in auth && "refreshToken" in auth && !("getAccessToken" in auth) ? createAuth(auth) : auth;
|
|
54
|
+
return ofetch.create({
|
|
55
|
+
...options,
|
|
56
|
+
timeout: options?.timeout ?? 3e4,
|
|
57
|
+
retry: 3,
|
|
58
|
+
retryDelay: (ctx) => {
|
|
59
|
+
const status = ctx.response?.status;
|
|
60
|
+
if (status === 429 || status === 503) {
|
|
61
|
+
const header = ctx.response?.headers.get("retry-after");
|
|
62
|
+
if (header) {
|
|
63
|
+
const secs = Number.parseInt(header, 10);
|
|
64
|
+
if (Number.isFinite(secs)) return secs * 1e3;
|
|
65
|
+
const when = Date.parse(header);
|
|
66
|
+
if (Number.isFinite(when)) return Math.max(0, when - Date.now());
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return 1e3;
|
|
70
|
+
},
|
|
71
|
+
retryStatusCodes: [
|
|
72
|
+
408,
|
|
73
|
+
425,
|
|
74
|
+
429,
|
|
75
|
+
500,
|
|
76
|
+
502,
|
|
77
|
+
503,
|
|
78
|
+
504
|
|
79
|
+
],
|
|
80
|
+
headers: {
|
|
81
|
+
...options?.headers,
|
|
82
|
+
"Accept-Encoding": "gzip",
|
|
83
|
+
"User-Agent": "gscdump (gzip)"
|
|
84
|
+
},
|
|
85
|
+
async onRequest({ options }) {
|
|
86
|
+
const token = await resolveToken(authState);
|
|
87
|
+
if (token) {
|
|
88
|
+
options.headers = new Headers(options.headers);
|
|
89
|
+
options.headers.set("Authorization", `Bearer ${token}`);
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
async onResponseError(ctx) {
|
|
93
|
+
if (ctx.response.status === 403) console.error("[gscdump] Permission denied (403). check your service account permissions being added to the GSC property.");
|
|
94
|
+
if (options?.onResponseError) if (Array.isArray(options.onResponseError)) for (const handler of options.onResponseError) await handler(ctx);
|
|
95
|
+
else await options.onResponseError(ctx);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
function googleSearchConsole(auth, options = {}) {
|
|
100
|
+
let fetch;
|
|
101
|
+
const authState = typeof auth === "object" && auth !== null && "clientId" in auth && "refreshToken" in auth && !("getAccessToken" in auth) ? createAuth(auth) : auth;
|
|
102
|
+
if (options.fetch) fetch = options.fetch;
|
|
103
|
+
else {
|
|
104
|
+
const fetchOptions = { ...options.fetchOptions };
|
|
105
|
+
if (options.onRateLimited) {
|
|
106
|
+
const originalOnError = fetchOptions.onResponseError;
|
|
107
|
+
fetchOptions.onResponseError = async (ctx) => {
|
|
108
|
+
if (ctx.response.status === 429) await options.onRateLimited({ response: ctx.response });
|
|
109
|
+
if (originalOnError) if (Array.isArray(originalOnError)) for (const handler of originalOnError) await handler(ctx);
|
|
110
|
+
else await originalOnError(ctx);
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
fetch = createFetch(authState, fetchOptions);
|
|
114
|
+
}
|
|
115
|
+
const querySearchAnalytics = (siteUrl, body, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/searchAnalytics/query`, {
|
|
116
|
+
method: "POST",
|
|
117
|
+
body,
|
|
118
|
+
signal: opts?.signal
|
|
119
|
+
});
|
|
120
|
+
return {
|
|
121
|
+
async *query(siteUrl, builder, opts) {
|
|
122
|
+
const state = builder.getState();
|
|
123
|
+
const body = resolveToBody(state);
|
|
124
|
+
const totalCap = body.rowLimit;
|
|
125
|
+
const pageSize = Math.min(totalCap ?? 25e3, 25e3);
|
|
126
|
+
let startRow = body.startRow || 0;
|
|
127
|
+
let yielded = 0;
|
|
128
|
+
let metadata;
|
|
129
|
+
let responseAggregationType;
|
|
130
|
+
while (true) {
|
|
131
|
+
opts?.signal?.throwIfAborted();
|
|
132
|
+
const remaining = totalCap ? totalCap - yielded : pageSize;
|
|
133
|
+
if (remaining <= 0) break;
|
|
134
|
+
const rowLimit = Math.min(pageSize, remaining);
|
|
135
|
+
const response = await querySearchAnalytics(siteUrl, {
|
|
136
|
+
...body,
|
|
137
|
+
startRow,
|
|
138
|
+
rowLimit
|
|
139
|
+
}, opts);
|
|
140
|
+
if (response.metadata) metadata = response.metadata;
|
|
141
|
+
if (response.responseAggregationType) responseAggregationType = response.responseAggregationType;
|
|
142
|
+
const rows = (response.rows || []).map((row) => {
|
|
143
|
+
const result = rowWithMetricDefaults(row);
|
|
144
|
+
state.dimensions.forEach((dim, i) => {
|
|
145
|
+
result[dim] = row.keys?.[i];
|
|
146
|
+
});
|
|
147
|
+
return result;
|
|
148
|
+
});
|
|
149
|
+
if (rows.length === 0) break;
|
|
150
|
+
yield rows;
|
|
151
|
+
yielded += rows.length;
|
|
152
|
+
startRow += rows.length;
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
metadata,
|
|
156
|
+
responseAggregationType
|
|
157
|
+
};
|
|
158
|
+
},
|
|
159
|
+
sites: (() => {
|
|
160
|
+
const list = async (opts) => {
|
|
161
|
+
return (await fetch(`${GSC_API}/webmasters/v3/sites`, { signal: opts?.signal })).siteEntry || [];
|
|
162
|
+
};
|
|
163
|
+
return Object.assign(list, {
|
|
164
|
+
list,
|
|
165
|
+
get: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}`, { signal: opts?.signal }),
|
|
166
|
+
add: (siteUrl, opts) => {
|
|
167
|
+
assertValidSiteUrl(siteUrl);
|
|
168
|
+
return fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}`, {
|
|
169
|
+
method: "PUT",
|
|
170
|
+
signal: opts?.signal
|
|
171
|
+
});
|
|
172
|
+
},
|
|
173
|
+
delete: (siteUrl, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}`, {
|
|
174
|
+
method: "DELETE",
|
|
175
|
+
signal: opts?.signal
|
|
176
|
+
})
|
|
177
|
+
});
|
|
178
|
+
})(),
|
|
179
|
+
verification: {
|
|
180
|
+
getToken: (params, opts) => fetch(`${SITE_VERIFICATION_API}/token`, {
|
|
181
|
+
method: "POST",
|
|
182
|
+
body: params,
|
|
183
|
+
signal: opts?.signal
|
|
184
|
+
}),
|
|
185
|
+
insert: (params, opts) => fetch(`${SITE_VERIFICATION_API}/webResource`, {
|
|
186
|
+
method: "POST",
|
|
187
|
+
query: { verificationMethod: params.verificationMethod },
|
|
188
|
+
body: { site: params.site },
|
|
189
|
+
signal: opts?.signal
|
|
190
|
+
}),
|
|
191
|
+
list: async (opts) => {
|
|
192
|
+
return (await fetch(`${SITE_VERIFICATION_API}/webResource`, { signal: opts?.signal })).items || [];
|
|
193
|
+
},
|
|
194
|
+
get: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, { signal: opts?.signal }),
|
|
195
|
+
delete: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
|
|
196
|
+
method: "DELETE",
|
|
197
|
+
signal: opts?.signal
|
|
198
|
+
})
|
|
199
|
+
},
|
|
200
|
+
inspect: (siteUrl, url, opts) => fetch(`${GSC_API}/v1/urlInspection/index:inspect`, {
|
|
201
|
+
method: "POST",
|
|
202
|
+
body: opts?.languageCode ? {
|
|
203
|
+
inspectionUrl: url,
|
|
204
|
+
siteUrl,
|
|
205
|
+
languageCode: opts.languageCode
|
|
206
|
+
} : {
|
|
207
|
+
inspectionUrl: url,
|
|
208
|
+
siteUrl
|
|
209
|
+
},
|
|
210
|
+
signal: opts?.signal
|
|
211
|
+
}),
|
|
212
|
+
sitemaps: {
|
|
213
|
+
list: async (siteUrl, opts) => {
|
|
214
|
+
return (await fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps`, {
|
|
215
|
+
signal: opts?.signal,
|
|
216
|
+
query: opts?.sitemapIndex ? { sitemapIndex: opts.sitemapIndex } : void 0
|
|
217
|
+
})).sitemap || [];
|
|
218
|
+
},
|
|
219
|
+
get: (siteUrl, feedpath, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`, { signal: opts?.signal }),
|
|
220
|
+
submit: (siteUrl, feedpath, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`, {
|
|
221
|
+
method: "PUT",
|
|
222
|
+
signal: opts?.signal
|
|
223
|
+
}),
|
|
224
|
+
delete: (siteUrl, feedpath, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/sitemaps/${encodeURIComponent(feedpath)}`, {
|
|
225
|
+
method: "DELETE",
|
|
226
|
+
signal: opts?.signal
|
|
227
|
+
})
|
|
228
|
+
},
|
|
229
|
+
indexing: {
|
|
230
|
+
publish: (url, type, opts) => fetch(`${INDEXING_API}/v3/urlNotifications:publish`, {
|
|
231
|
+
method: "POST",
|
|
232
|
+
body: {
|
|
233
|
+
url,
|
|
234
|
+
type
|
|
235
|
+
},
|
|
236
|
+
signal: opts?.signal
|
|
237
|
+
}),
|
|
238
|
+
getMetadata: (url, opts) => fetch(`${INDEXING_API}/v3/urlNotifications/metadata`, {
|
|
239
|
+
query: { url },
|
|
240
|
+
signal: opts?.signal
|
|
241
|
+
})
|
|
242
|
+
},
|
|
243
|
+
searchAnalytics: { query: querySearchAnalytics }
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
export { DEFAULT_GSC_REQUEST_TIMEOUT_MS, createAuth, createFetch, googleSearchConsole };
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
type GscErrorKind = 'auth-expired' | 'rate-limited' | 'not-found' | 'validation' | 'storage' | 'transport';
|
|
2
|
+
type GscError = {
|
|
3
|
+
kind: 'auth-expired';
|
|
4
|
+
message: string;
|
|
5
|
+
cause: unknown;
|
|
6
|
+
} | {
|
|
7
|
+
kind: 'rate-limited';
|
|
8
|
+
message: string;
|
|
9
|
+
retryAfter?: number;
|
|
10
|
+
cause: unknown;
|
|
11
|
+
} | {
|
|
12
|
+
kind: 'not-found';
|
|
13
|
+
message: string;
|
|
14
|
+
cause: unknown;
|
|
15
|
+
} | {
|
|
16
|
+
kind: 'validation';
|
|
17
|
+
message: string;
|
|
18
|
+
cause: unknown;
|
|
19
|
+
} | {
|
|
20
|
+
kind: 'storage';
|
|
21
|
+
message: string;
|
|
22
|
+
cause: unknown;
|
|
23
|
+
} | {
|
|
24
|
+
kind: 'transport';
|
|
25
|
+
message: string;
|
|
26
|
+
status?: number;
|
|
27
|
+
cause: unknown;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Classify an unknown error into a `GscError` discriminated union.
|
|
31
|
+
* Transport is the catch-all — anything without a recognizable status ends up there.
|
|
32
|
+
*/
|
|
33
|
+
declare function classifyError(cause: unknown): GscError;
|
|
34
|
+
/**
|
|
35
|
+
* Re-raises a `GscError` value as an `Error`, preserving its `cause` for
|
|
36
|
+
* stack-walking and stashing the original union under `.gscError`. Pairs with
|
|
37
|
+
* `unwrapResult` from `gscdump/result` so a `fooResult(): Result<A, GscError>`
|
|
38
|
+
* core can be wrapped by a throwing `foo()` without losing the typed error.
|
|
39
|
+
*/
|
|
40
|
+
declare function gscErrorToException(error: GscError): Error;
|
|
41
|
+
interface GscApiErrorInfo {
|
|
42
|
+
code: number;
|
|
43
|
+
message: string;
|
|
44
|
+
reason?: string;
|
|
45
|
+
status?: string;
|
|
46
|
+
}
|
|
47
|
+
declare function parseGoogleError(text: string, httpStatus?: number): GscApiErrorInfo;
|
|
48
|
+
declare class GscApiError extends Error {
|
|
49
|
+
info: GscApiErrorInfo;
|
|
50
|
+
constructor(message: string, info: GscApiErrorInfo);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Returns a handler that re-throws `unknown` as a `GscApiError` prefixed with
|
|
54
|
+
* `prefix`. Recognises `ofetch` `FetchError` (parses `err.data` as Google JSON
|
|
55
|
+
* via `parseGoogleError`); passes through existing `GscApiError`; re-throws
|
|
56
|
+
* anything else as-is.
|
|
57
|
+
*/
|
|
58
|
+
declare function rethrowAsGscApiError(prefix: string): (err: unknown) => never;
|
|
59
|
+
/**
|
|
60
|
+
* String-matches an error against the signals Google returns when a token
|
|
61
|
+
* has lost access to a property (revoked, downgraded, or never had it).
|
|
62
|
+
* Cheaper than a full {@link classifyError} call when the caller only
|
|
63
|
+
* needs the permission verdict.
|
|
64
|
+
*/
|
|
65
|
+
declare function isPermissionDeniedError(err: unknown): boolean;
|
|
66
|
+
export { GscApiError, GscApiErrorInfo, GscError, GscErrorKind, classifyError, gscErrorToException, isPermissionDeniedError, parseGoogleError, rethrowAsGscApiError };
|