gscdump 2.1.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/dist/bing/client.d.mts +5 -0
- package/dist/bing/client.mjs +215 -0
- package/dist/bing/index.d.mts +4 -0
- package/dist/bing/index.mjs +3 -0
- package/dist/bing/normalize.d.mts +9 -0
- package/dist/bing/normalize.mjs +124 -0
- package/dist/bing/types.d.mts +181 -0
- package/dist/query/plan.d.mts +1 -1
- package/dist/query/resolver.d.mts +1 -1
- package/package.json +9 -3
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://npm.chart.dev/gscdump)
|
|
5
5
|
[](https://github.com/harlan-zw/gscdump/blob/main/LICENSE)
|
|
6
6
|
|
|
7
|
-
Direct Google Search Console
|
|
7
|
+
Direct Google Search Console and Bing Webmaster clients with typed queries and Indexing Evidence.
|
|
8
8
|
|
|
9
9
|
## Install
|
|
10
10
|
|
|
@@ -73,6 +73,24 @@ await client.indexing.publish(
|
|
|
73
73
|
|
|
74
74
|
The package root also exports batch and projection helpers such as `fetchSitesWithSitemaps`, `batchInspectUrlsFlatSettled`, `inspectUrlFlat`, and `batchRequestIndexing`.
|
|
75
75
|
|
|
76
|
+
## Read Bing Indexing Evidence
|
|
77
|
+
|
|
78
|
+
Use `gscdump/bing` with an OAuth access token. The client returns tagged
|
|
79
|
+
`Result` values and never infers an indexed verdict from crawl evidence.
|
|
80
|
+
|
|
81
|
+
```ts
|
|
82
|
+
import { bingWebmaster } from 'gscdump/bing'
|
|
83
|
+
|
|
84
|
+
const client = bingWebmaster({ accessToken: 'access-token' })
|
|
85
|
+
const evidence = await client.getIndexingEvidence(
|
|
86
|
+
'https://example.com/',
|
|
87
|
+
'https://example.com/docs',
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
if (evidence.ok)
|
|
91
|
+
console.log(evidence.value)
|
|
92
|
+
```
|
|
93
|
+
|
|
76
94
|
Sitemap XML reading and traversal lives in `sitemapd`. Product feed scoping and
|
|
77
95
|
exact membership hashing live in `gscdump/sitemap-identity`. Hosted canonical
|
|
78
96
|
sitemap membership is available through `@gscdump/sdk/v1`.
|
|
@@ -81,6 +99,7 @@ sitemap membership is available through `@gscdump/sdk/v1`.
|
|
|
81
99
|
|
|
82
100
|
- `gscdump/query`: query builder, columns, operators, Pacific date helpers, and logical query plans
|
|
83
101
|
- `gscdump/query/plan`: logical query planning only
|
|
102
|
+
- `gscdump/bing`: Bing Site, URL, traffic, and crawl evidence calls
|
|
84
103
|
- `gscdump/dates`: explicit UTC and Pacific Search Console date helpers
|
|
85
104
|
- `gscdump/contracts`: Search Analytics request and response contracts
|
|
86
105
|
- `gscdump/result`: `Result` helpers
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { BingWebmasterClient, BingWebmasterOptions } from "./types.mjs";
|
|
2
|
+
declare const DEFAULT_BING_WEBMASTER_API_URL = "https://www.bing.com/webmaster/api.svc/json";
|
|
3
|
+
declare const DEFAULT_BING_CHILD_PAGE_LIMIT = 100;
|
|
4
|
+
declare function bingWebmaster(options: BingWebmasterOptions): BingWebmasterClient;
|
|
5
|
+
export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster };
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { err, ok } from "../core/result.mjs";
|
|
2
|
+
import { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
3
|
+
const DEFAULT_BING_WEBMASTER_API_URL = "https://www.bing.com/webmaster/api.svc/json";
|
|
4
|
+
const DEFAULT_BING_CHILD_PAGE_LIMIT = 100;
|
|
5
|
+
const BING_MAX_CHILD_PAGE_COUNT = 65536;
|
|
6
|
+
function isRecord(value) {
|
|
7
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
8
|
+
}
|
|
9
|
+
function parseErrorPayload(value) {
|
|
10
|
+
if (!isRecord(value)) return {};
|
|
11
|
+
return {
|
|
12
|
+
...typeof value.ErrorCode === "number" ? { ErrorCode: value.ErrorCode } : {},
|
|
13
|
+
...typeof value.Message === "string" ? { Message: value.Message } : {}
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function retryAfterMs(response, now) {
|
|
17
|
+
const header = response.headers.get("retry-after");
|
|
18
|
+
if (!header) return void 0;
|
|
19
|
+
const seconds = Number(header);
|
|
20
|
+
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1e3;
|
|
21
|
+
const date = Date.parse(header);
|
|
22
|
+
if (!Number.isFinite(date)) return void 0;
|
|
23
|
+
return Math.max(0, date - now.getTime());
|
|
24
|
+
}
|
|
25
|
+
function mapResponseError(response, payload, now) {
|
|
26
|
+
const apiError = parseErrorPayload(payload);
|
|
27
|
+
if (response.status === 401 || apiError.ErrorCode === 3) return { _tag: "AuthenticationRequired" };
|
|
28
|
+
if (apiError.ErrorCode === 6) return { _tag: "UserBlocked" };
|
|
29
|
+
if (response.status === 403 || apiError.ErrorCode === 14) return { _tag: "PermissionDenied" };
|
|
30
|
+
if (response.status === 429 || apiError.ErrorCode === 4 || apiError.ErrorCode === 5) {
|
|
31
|
+
const delay = retryAfterMs(response, now);
|
|
32
|
+
return {
|
|
33
|
+
_tag: "Throttled",
|
|
34
|
+
...delay === void 0 ? {} : { retryAfterMs: delay }
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
if (response.status >= 500) return {
|
|
38
|
+
_tag: "ProviderUnavailable",
|
|
39
|
+
status: response.status
|
|
40
|
+
};
|
|
41
|
+
return {
|
|
42
|
+
_tag: "RequestRejected",
|
|
43
|
+
...apiError.ErrorCode === void 0 ? {} : { errorCode: apiError.ErrorCode },
|
|
44
|
+
...apiError.Message === void 0 ? {} : { message: apiError.Message },
|
|
45
|
+
status: response.status
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function parseWrapped(value, parser) {
|
|
49
|
+
if (!isRecord(value) || !Object.hasOwn(value, "d")) return err("invalid-payload");
|
|
50
|
+
return parser(value.d);
|
|
51
|
+
}
|
|
52
|
+
function parseNullable(parser) {
|
|
53
|
+
return (value) => {
|
|
54
|
+
if (value === null) return ok(null);
|
|
55
|
+
return parser(value);
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function parseList(parser) {
|
|
59
|
+
return (value) => {
|
|
60
|
+
if (!Array.isArray(value)) return err("invalid-payload");
|
|
61
|
+
const parsed = [];
|
|
62
|
+
for (const item of value) {
|
|
63
|
+
const result = parser(item);
|
|
64
|
+
if (!result.ok) return result;
|
|
65
|
+
parsed.push(result.value);
|
|
66
|
+
}
|
|
67
|
+
return ok(parsed);
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const CRAWL_DATE_FILTERS = {
|
|
71
|
+
"any": 0,
|
|
72
|
+
"last-week": 1,
|
|
73
|
+
"last-two-weeks": 2,
|
|
74
|
+
"last-three-weeks": 4
|
|
75
|
+
};
|
|
76
|
+
const DISCOVERED_DATE_FILTERS = {
|
|
77
|
+
"any": 0,
|
|
78
|
+
"last-week": 1,
|
|
79
|
+
"last-month": 2
|
|
80
|
+
};
|
|
81
|
+
const DOCUMENT_FILTERS = {
|
|
82
|
+
"any": 0,
|
|
83
|
+
"blocked-by-robots-txt": 1,
|
|
84
|
+
"malware": 2
|
|
85
|
+
};
|
|
86
|
+
const HTTP_CODE_FILTERS = {
|
|
87
|
+
"any": 0,
|
|
88
|
+
"2xx": 1,
|
|
89
|
+
"3xx": 2,
|
|
90
|
+
"301": 4,
|
|
91
|
+
"302": 8,
|
|
92
|
+
"4xx": 16,
|
|
93
|
+
"5xx": 32,
|
|
94
|
+
"other": 64
|
|
95
|
+
};
|
|
96
|
+
function toFilterProperties(filters = {}) {
|
|
97
|
+
return {
|
|
98
|
+
__type: "FilterProperties:#Microsoft.Bing.Webmaster.Api",
|
|
99
|
+
CrawlDateFilter: CRAWL_DATE_FILTERS[filters.crawlDate ?? "any"],
|
|
100
|
+
DiscoveredDateFilter: DISCOVERED_DATE_FILTERS[filters.discoveredDate ?? "any"],
|
|
101
|
+
DocFlagsFilters: DOCUMENT_FILTERS[filters.document ?? "any"],
|
|
102
|
+
HttpCodeFilters: HTTP_CODE_FILTERS[filters.httpCode ?? "any"]
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function bingWebmaster(options) {
|
|
106
|
+
const baseUrl = (options.baseUrl ?? "https://www.bing.com/webmaster/api.svc/json").replace(/\/+$/, "");
|
|
107
|
+
const clock = options.clock ?? (() => /* @__PURE__ */ new Date());
|
|
108
|
+
const fetch = options.fetch ?? globalThis.fetch.bind(globalThis);
|
|
109
|
+
const request = async (operation, parser, input = {}) => {
|
|
110
|
+
if (!options.accessToken.trim()) return err({ _tag: "AuthenticationRequired" });
|
|
111
|
+
const url = new URL(`${baseUrl}/${operation}`);
|
|
112
|
+
for (const [key, value] of Object.entries(input.query ?? {})) url.searchParams.set(key, value);
|
|
113
|
+
const response = await fetch(url.toString(), {
|
|
114
|
+
...input.body === void 0 ? {} : { body: JSON.stringify(input.body) },
|
|
115
|
+
headers: {
|
|
116
|
+
Accept: "application/json",
|
|
117
|
+
Authorization: `Bearer ${options.accessToken}`,
|
|
118
|
+
...input.body === void 0 ? {} : { "Content-Type": "application/json; charset=utf-8" }
|
|
119
|
+
},
|
|
120
|
+
method: input.method ?? "GET",
|
|
121
|
+
signal: input.signal
|
|
122
|
+
});
|
|
123
|
+
const payload = await response.json().then((value) => ok(value)).catch(() => err("invalid-json"));
|
|
124
|
+
if (!response.ok) return err(mapResponseError(response, payload.ok ? payload.value : void 0, clock()));
|
|
125
|
+
if (!payload.ok) return err({
|
|
126
|
+
_tag: "MalformedResponse",
|
|
127
|
+
operation,
|
|
128
|
+
reason: "invalid-json"
|
|
129
|
+
});
|
|
130
|
+
const parsed = parseWrapped(payload.value, parser);
|
|
131
|
+
if (!parsed.ok) return err({
|
|
132
|
+
_tag: "MalformedResponse",
|
|
133
|
+
operation,
|
|
134
|
+
reason: "invalid-payload"
|
|
135
|
+
});
|
|
136
|
+
return parsed;
|
|
137
|
+
};
|
|
138
|
+
const getUserSites = (callOptions) => request("GetUserSites", parseList(normalizeBingSite), { signal: callOptions?.signal });
|
|
139
|
+
const getUrlInfo = (siteUrl, url, callOptions) => request("GetUrlInfo", parseNullable(normalizeBingUrlInfo), {
|
|
140
|
+
query: {
|
|
141
|
+
siteUrl,
|
|
142
|
+
url
|
|
143
|
+
},
|
|
144
|
+
signal: callOptions?.signal
|
|
145
|
+
});
|
|
146
|
+
return {
|
|
147
|
+
getUserSites,
|
|
148
|
+
async getVerifiedSite(siteUrl, callOptions) {
|
|
149
|
+
const sites = await getUserSites(callOptions);
|
|
150
|
+
if (!sites.ok) return sites;
|
|
151
|
+
const site = sites.value.find((candidate) => candidate.url === siteUrl);
|
|
152
|
+
if (!site) return err({
|
|
153
|
+
_tag: "SiteUnavailable",
|
|
154
|
+
siteUrl
|
|
155
|
+
});
|
|
156
|
+
if (!site.isVerified) return err({
|
|
157
|
+
_tag: "UnverifiedSite",
|
|
158
|
+
siteUrl
|
|
159
|
+
});
|
|
160
|
+
return ok(site);
|
|
161
|
+
},
|
|
162
|
+
getUrlInfo,
|
|
163
|
+
async getIndexingEvidence(siteUrl, url, callOptions) {
|
|
164
|
+
const info = await getUrlInfo(siteUrl, url, callOptions);
|
|
165
|
+
if (!info.ok) return info;
|
|
166
|
+
return normalizeBingIndexingEvidence(info.value, url, clock());
|
|
167
|
+
},
|
|
168
|
+
getUrlTrafficInfo: (siteUrl, url, callOptions) => request("GetUrlTrafficInfo", parseNullable(normalizeBingUrlTrafficInfo), {
|
|
169
|
+
query: {
|
|
170
|
+
siteUrl,
|
|
171
|
+
url
|
|
172
|
+
},
|
|
173
|
+
signal: callOptions?.signal
|
|
174
|
+
}),
|
|
175
|
+
async getChildrenUrlInfo(siteUrl, url, childrenOptions = {}) {
|
|
176
|
+
const maxPages = childrenOptions.maxPages ?? 100;
|
|
177
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > BING_MAX_CHILD_PAGE_COUNT) return err({
|
|
178
|
+
_tag: "InvalidPagination",
|
|
179
|
+
maximum: BING_MAX_CHILD_PAGE_COUNT,
|
|
180
|
+
maxPages,
|
|
181
|
+
minimum: 1,
|
|
182
|
+
reason: "max-pages-out-of-range"
|
|
183
|
+
});
|
|
184
|
+
const children = [];
|
|
185
|
+
for (let page = 0; page < maxPages; page++) {
|
|
186
|
+
const response = await request("GetChildrenUrlInfo", parseList(normalizeBingUrlInfo), {
|
|
187
|
+
body: {
|
|
188
|
+
filterProperties: toFilterProperties(childrenOptions.filters),
|
|
189
|
+
page,
|
|
190
|
+
siteUrl,
|
|
191
|
+
url
|
|
192
|
+
},
|
|
193
|
+
method: "POST",
|
|
194
|
+
signal: childrenOptions.signal
|
|
195
|
+
});
|
|
196
|
+
if (!response.ok) return response;
|
|
197
|
+
if (response.value.length === 0) return ok(children);
|
|
198
|
+
children.push(...response.value);
|
|
199
|
+
}
|
|
200
|
+
return err({
|
|
201
|
+
_tag: "PaginationLimitExceeded",
|
|
202
|
+
maxPages
|
|
203
|
+
});
|
|
204
|
+
},
|
|
205
|
+
getPageStats: (siteUrl, callOptions) => request("GetPageStats", parseList(normalizeBingPageStats), {
|
|
206
|
+
query: { siteUrl },
|
|
207
|
+
signal: callOptions?.signal
|
|
208
|
+
}),
|
|
209
|
+
getCrawlIssues: (siteUrl, callOptions) => request("GetCrawlIssues", parseList(normalizeBingCrawlIssue), {
|
|
210
|
+
query: { siteUrl },
|
|
211
|
+
signal: callOptions?.signal
|
|
212
|
+
})
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster };
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { BingCallOptions, BingChildrenFilters, BingChildrenOptions, BingCrawlDateFilter, BingCrawlEvidence, BingCrawlIssue, BingCrawlIssueWire, BingDiscoveredDateFilter, BingDocumentFilter, BingEvidenceError, BingFetch, BingHttpCodeFilter, BingIndexingEvidence, BingOperation, BingPageStats, BingProviderError, BingQueryStatsWire, BingSite, BingSiteWire, BingUnknownEvidence, BingUrlInfo, BingUrlInfoWire, BingUrlTrafficInfo, BingUrlTrafficInfoWire, BingUrlWithCrawlIssues, BingWebmasterClient, BingWebmasterOptions, BingWireResponse } from "./types.mjs";
|
|
2
|
+
import { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster } from "./client.mjs";
|
|
3
|
+
import { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
4
|
+
export { type BingCallOptions, type BingChildrenFilters, type BingChildrenOptions, type BingCrawlDateFilter, type BingCrawlEvidence, type BingCrawlIssue, type BingCrawlIssueWire, type BingDiscoveredDateFilter, type BingDocumentFilter, type BingEvidenceError, type BingFetch, type BingHttpCodeFilter, type BingIndexingEvidence, type BingOperation, type BingPageStats, type BingProviderError, type BingQueryStatsWire, type BingSite, type BingSiteWire, type BingUnknownEvidence, type BingUrlInfo, type BingUrlInfoWire, type BingUrlTrafficInfo, type BingUrlTrafficInfoWire, type BingUrlWithCrawlIssues, type BingWebmasterClient, type BingWebmasterOptions, type BingWireResponse, DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
2
|
+
import { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster } from "./client.mjs";
|
|
3
|
+
export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Result } from "../core/result.mjs";
|
|
2
|
+
import { BingEvidenceError, BingIndexingEvidence, BingPageStats, BingSite, BingUrlInfo, BingUrlTrafficInfo, BingUrlWithCrawlIssues } from "./types.mjs";
|
|
3
|
+
declare function normalizeBingSite(value: unknown): Result<BingSite, 'invalid-payload'>;
|
|
4
|
+
declare function normalizeBingUrlInfo(value: unknown): Result<BingUrlInfo, 'invalid-payload'>;
|
|
5
|
+
declare function normalizeBingUrlTrafficInfo(value: unknown): Result<BingUrlTrafficInfo, 'invalid-payload'>;
|
|
6
|
+
declare function normalizeBingPageStats(value: unknown): Result<BingPageStats, 'invalid-payload'>;
|
|
7
|
+
declare function normalizeBingCrawlIssue(value: unknown): Result<BingUrlWithCrawlIssues, 'invalid-payload'>;
|
|
8
|
+
declare function normalizeBingIndexingEvidence(info: BingUrlInfo | null, url: string, observedAt: Date): Result<BingIndexingEvidence, BingEvidenceError>;
|
|
9
|
+
export { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { err, ok } from "../core/result.mjs";
|
|
2
|
+
const BING_UNAVAILABLE_DATE_MS = -62135568e6;
|
|
3
|
+
const BING_DATE_PATTERN = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/;
|
|
4
|
+
function isRecord(value) {
|
|
5
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function isNonNegativeInteger(value) {
|
|
8
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
9
|
+
}
|
|
10
|
+
function parseBingDate(value) {
|
|
11
|
+
if (value === void 0 || value === null) return ok(void 0);
|
|
12
|
+
if (typeof value !== "string") return err("invalid-date");
|
|
13
|
+
const match = BING_DATE_PATTERN.exec(value);
|
|
14
|
+
if (!match) return err("invalid-date");
|
|
15
|
+
const milliseconds = Number(match[1]);
|
|
16
|
+
if (!Number.isFinite(milliseconds)) return err("invalid-date");
|
|
17
|
+
if (milliseconds <= BING_UNAVAILABLE_DATE_MS) return ok(void 0);
|
|
18
|
+
const date = new Date(milliseconds);
|
|
19
|
+
if (!Number.isFinite(date.getTime())) return err("invalid-date");
|
|
20
|
+
return ok(date.toISOString());
|
|
21
|
+
}
|
|
22
|
+
function normalizeBingSite(value) {
|
|
23
|
+
if (!isRecord(value) || typeof value.IsVerified !== "boolean" || typeof value.Url !== "string") return err("invalid-payload");
|
|
24
|
+
return ok({
|
|
25
|
+
isVerified: value.IsVerified,
|
|
26
|
+
url: value.Url
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
function normalizeBingUrlInfo(value) {
|
|
30
|
+
if (!isRecord(value) || !isNonNegativeInteger(value.AnchorCount) || !isNonNegativeInteger(value.DocumentSize) || !isNonNegativeInteger(value.HttpStatus) || typeof value.IsPage !== "boolean" || !isNonNegativeInteger(value.TotalChildUrlCount) || typeof value.Url !== "string") return err("invalid-payload");
|
|
31
|
+
const discoveryDate = parseBingDate(value.DiscoveryDate);
|
|
32
|
+
const lastCrawledDate = parseBingDate(value.LastCrawledDate);
|
|
33
|
+
if (!discoveryDate.ok || !lastCrawledDate.ok) return err("invalid-payload");
|
|
34
|
+
return ok({
|
|
35
|
+
anchorCount: value.AnchorCount,
|
|
36
|
+
...discoveryDate.value ? { discoveryDate: discoveryDate.value } : {},
|
|
37
|
+
documentSize: value.DocumentSize,
|
|
38
|
+
httpStatus: value.HttpStatus,
|
|
39
|
+
isPage: value.IsPage,
|
|
40
|
+
...lastCrawledDate.value ? { lastCrawledDate: lastCrawledDate.value } : {},
|
|
41
|
+
totalChildUrlCount: value.TotalChildUrlCount,
|
|
42
|
+
url: value.Url
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
function normalizeBingUrlTrafficInfo(value) {
|
|
46
|
+
if (!isRecord(value) || !isNonNegativeInteger(value.Clicks) || !isNonNegativeInteger(value.Impressions) || typeof value.IsPage !== "boolean" || typeof value.Url !== "string") return err("invalid-payload");
|
|
47
|
+
return ok({
|
|
48
|
+
clicks: value.Clicks,
|
|
49
|
+
impressions: value.Impressions,
|
|
50
|
+
isPage: value.IsPage,
|
|
51
|
+
url: value.Url
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
function normalizeBingPageStats(value) {
|
|
55
|
+
if (!isRecord(value) || !isNonNegativeInteger(value.AvgClickPosition) || !isNonNegativeInteger(value.AvgImpressionPosition) || !isNonNegativeInteger(value.Clicks) || !isNonNegativeInteger(value.Impressions) || typeof value.Query !== "string") return err("invalid-payload");
|
|
56
|
+
const date = parseBingDate(value.Date);
|
|
57
|
+
if (!date.ok || !date.value) return err("invalid-payload");
|
|
58
|
+
return ok({
|
|
59
|
+
averageClickPosition: value.AvgClickPosition,
|
|
60
|
+
averageImpressionPosition: value.AvgImpressionPosition,
|
|
61
|
+
clicks: value.Clicks,
|
|
62
|
+
date: date.value,
|
|
63
|
+
impressions: value.Impressions,
|
|
64
|
+
query: value.Query
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
const CRAWL_ISSUES = {
|
|
68
|
+
0: "none",
|
|
69
|
+
1: "301",
|
|
70
|
+
2: "302",
|
|
71
|
+
4: "4xx",
|
|
72
|
+
8: "5xx",
|
|
73
|
+
16: "blocked-by-robots-txt",
|
|
74
|
+
32: "contains-malware",
|
|
75
|
+
64: "important-url-blocked-by-robots-txt",
|
|
76
|
+
128: "dns-errors",
|
|
77
|
+
256: "timeout-errors"
|
|
78
|
+
};
|
|
79
|
+
function normalizeBingCrawlIssue(value) {
|
|
80
|
+
if (!isRecord(value) || !isNonNegativeInteger(value.HttpCode) || !isNonNegativeInteger(value.InLinks) || !isNonNegativeInteger(value.Issues) || typeof value.Url !== "string") return err("invalid-payload");
|
|
81
|
+
return ok({
|
|
82
|
+
httpCode: value.HttpCode,
|
|
83
|
+
inLinks: value.InLinks,
|
|
84
|
+
issue: CRAWL_ISSUES[value.Issues] ?? "unknown",
|
|
85
|
+
rawIssueCode: value.Issues,
|
|
86
|
+
url: value.Url
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
function normalizeBingIndexingEvidence(info, url, observedAt) {
|
|
90
|
+
if (!info) return ok({
|
|
91
|
+
_tag: "UnknownEvidence",
|
|
92
|
+
observedAt: observedAt.toISOString(),
|
|
93
|
+
reason: "not-observed",
|
|
94
|
+
searchEngine: "bing",
|
|
95
|
+
url
|
|
96
|
+
});
|
|
97
|
+
if (!info.discoveryDate && !info.lastCrawledDate && info.httpStatus === 0 && info.documentSize === 0 && info.anchorCount === 0 && info.totalChildUrlCount === 0) return ok({
|
|
98
|
+
_tag: "UnknownEvidence",
|
|
99
|
+
observedAt: observedAt.toISOString(),
|
|
100
|
+
reason: "not-discovered",
|
|
101
|
+
searchEngine: "bing",
|
|
102
|
+
url
|
|
103
|
+
});
|
|
104
|
+
if (!info.isPage) return err({
|
|
105
|
+
_tag: "UnsupportedEvidence",
|
|
106
|
+
reason: "not-a-page",
|
|
107
|
+
url
|
|
108
|
+
});
|
|
109
|
+
return ok({
|
|
110
|
+
_tag: "CrawlEvidence",
|
|
111
|
+
anchorCount: info.anchorCount,
|
|
112
|
+
...info.discoveryDate ? { discoveryDate: info.discoveryDate } : {},
|
|
113
|
+
documentSize: info.documentSize,
|
|
114
|
+
httpStatus: info.httpStatus,
|
|
115
|
+
...info.lastCrawledDate ? { lastCrawledDate: info.lastCrawledDate } : {},
|
|
116
|
+
observedAt: observedAt.toISOString(),
|
|
117
|
+
rawStatus: info.httpStatus,
|
|
118
|
+
searchEngine: "bing",
|
|
119
|
+
totalChildUrlCount: info.totalChildUrlCount,
|
|
120
|
+
uncertaintyReason: "indexed-verdict-unavailable",
|
|
121
|
+
url
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
export { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { Result } from "../core/result.mjs";
|
|
2
|
+
interface BingSiteWire {
|
|
3
|
+
__type?: string;
|
|
4
|
+
AuthenticationCode: string;
|
|
5
|
+
DnsVerificationCode: string;
|
|
6
|
+
IsVerified: boolean;
|
|
7
|
+
Url: string;
|
|
8
|
+
}
|
|
9
|
+
interface BingUrlInfoWire {
|
|
10
|
+
__type?: string;
|
|
11
|
+
AnchorCount: number;
|
|
12
|
+
DiscoveryDate?: string | null;
|
|
13
|
+
DocumentSize: number;
|
|
14
|
+
HttpStatus: number;
|
|
15
|
+
IsPage: boolean;
|
|
16
|
+
LastCrawledDate?: string | null;
|
|
17
|
+
TotalChildUrlCount: number;
|
|
18
|
+
Url: string;
|
|
19
|
+
}
|
|
20
|
+
interface BingUrlTrafficInfoWire {
|
|
21
|
+
__type?: string;
|
|
22
|
+
Clicks: number;
|
|
23
|
+
Impressions: number;
|
|
24
|
+
IsPage: boolean;
|
|
25
|
+
Url: string;
|
|
26
|
+
}
|
|
27
|
+
interface BingQueryStatsWire {
|
|
28
|
+
__type?: string;
|
|
29
|
+
AvgClickPosition: number;
|
|
30
|
+
AvgImpressionPosition: number;
|
|
31
|
+
Clicks: number;
|
|
32
|
+
Date: string;
|
|
33
|
+
Impressions: number;
|
|
34
|
+
Query: string;
|
|
35
|
+
}
|
|
36
|
+
interface BingCrawlIssueWire {
|
|
37
|
+
__type?: string;
|
|
38
|
+
HttpCode: number;
|
|
39
|
+
InLinks: number;
|
|
40
|
+
Issues: number;
|
|
41
|
+
Url: string;
|
|
42
|
+
}
|
|
43
|
+
interface BingWireResponse<T> {
|
|
44
|
+
d: T;
|
|
45
|
+
}
|
|
46
|
+
interface BingSite {
|
|
47
|
+
isVerified: boolean;
|
|
48
|
+
url: string;
|
|
49
|
+
}
|
|
50
|
+
interface BingUrlInfo {
|
|
51
|
+
anchorCount: number;
|
|
52
|
+
discoveryDate?: string;
|
|
53
|
+
documentSize: number;
|
|
54
|
+
httpStatus: number;
|
|
55
|
+
isPage: boolean;
|
|
56
|
+
lastCrawledDate?: string;
|
|
57
|
+
totalChildUrlCount: number;
|
|
58
|
+
url: string;
|
|
59
|
+
}
|
|
60
|
+
interface BingUrlTrafficInfo {
|
|
61
|
+
clicks: number;
|
|
62
|
+
impressions: number;
|
|
63
|
+
isPage: boolean;
|
|
64
|
+
url: string;
|
|
65
|
+
}
|
|
66
|
+
interface BingPageStats {
|
|
67
|
+
averageClickPosition: number;
|
|
68
|
+
averageImpressionPosition: number;
|
|
69
|
+
clicks: number;
|
|
70
|
+
date: string;
|
|
71
|
+
impressions: number;
|
|
72
|
+
query: string;
|
|
73
|
+
}
|
|
74
|
+
type BingCrawlIssue = 'none' | '301' | '302' | '4xx' | '5xx' | 'blocked-by-robots-txt' | 'contains-malware' | 'important-url-blocked-by-robots-txt' | 'dns-errors' | 'timeout-errors' | 'unknown';
|
|
75
|
+
interface BingUrlWithCrawlIssues {
|
|
76
|
+
httpCode: number;
|
|
77
|
+
inLinks: number;
|
|
78
|
+
issue: BingCrawlIssue;
|
|
79
|
+
rawIssueCode: number;
|
|
80
|
+
url: string;
|
|
81
|
+
}
|
|
82
|
+
interface BingCrawlEvidence {
|
|
83
|
+
_tag: 'CrawlEvidence';
|
|
84
|
+
anchorCount: number;
|
|
85
|
+
discoveryDate?: string;
|
|
86
|
+
documentSize: number;
|
|
87
|
+
httpStatus: number;
|
|
88
|
+
lastCrawledDate?: string;
|
|
89
|
+
observedAt: string;
|
|
90
|
+
rawStatus: number;
|
|
91
|
+
searchEngine: 'bing';
|
|
92
|
+
totalChildUrlCount: number;
|
|
93
|
+
uncertaintyReason: 'indexed-verdict-unavailable';
|
|
94
|
+
url: string;
|
|
95
|
+
}
|
|
96
|
+
interface BingUnknownEvidence {
|
|
97
|
+
_tag: 'UnknownEvidence';
|
|
98
|
+
observedAt: string;
|
|
99
|
+
reason: 'not-discovered' | 'not-observed';
|
|
100
|
+
searchEngine: 'bing';
|
|
101
|
+
url: string;
|
|
102
|
+
}
|
|
103
|
+
type BingIndexingEvidence = BingCrawlEvidence | BingUnknownEvidence;
|
|
104
|
+
type BingProviderError = {
|
|
105
|
+
_tag: 'AuthenticationRequired';
|
|
106
|
+
} | {
|
|
107
|
+
_tag: 'PermissionDenied';
|
|
108
|
+
} | {
|
|
109
|
+
_tag: 'UserBlocked';
|
|
110
|
+
} | {
|
|
111
|
+
_tag: 'UnverifiedSite';
|
|
112
|
+
siteUrl: string;
|
|
113
|
+
} | {
|
|
114
|
+
_tag: 'SiteUnavailable';
|
|
115
|
+
siteUrl: string;
|
|
116
|
+
} | {
|
|
117
|
+
_tag: 'Throttled';
|
|
118
|
+
retryAfterMs?: number;
|
|
119
|
+
} | {
|
|
120
|
+
_tag: 'ProviderUnavailable';
|
|
121
|
+
status: number;
|
|
122
|
+
} | {
|
|
123
|
+
_tag: 'RequestRejected';
|
|
124
|
+
errorCode?: number;
|
|
125
|
+
message?: string;
|
|
126
|
+
status: number;
|
|
127
|
+
} | {
|
|
128
|
+
_tag: 'MalformedResponse';
|
|
129
|
+
operation: BingOperation;
|
|
130
|
+
reason: 'invalid-json' | 'invalid-payload';
|
|
131
|
+
} | {
|
|
132
|
+
_tag: 'InvalidPagination';
|
|
133
|
+
maximum: number;
|
|
134
|
+
maxPages: number;
|
|
135
|
+
minimum: number;
|
|
136
|
+
reason: 'max-pages-out-of-range';
|
|
137
|
+
} | {
|
|
138
|
+
_tag: 'PaginationLimitExceeded';
|
|
139
|
+
maxPages: number;
|
|
140
|
+
};
|
|
141
|
+
interface BingEvidenceError {
|
|
142
|
+
_tag: 'UnsupportedEvidence';
|
|
143
|
+
reason: 'not-a-page';
|
|
144
|
+
url: string;
|
|
145
|
+
}
|
|
146
|
+
type BingOperation = 'GetUserSites' | 'GetUrlInfo' | 'GetUrlTrafficInfo' | 'GetChildrenUrlInfo' | 'GetPageStats' | 'GetCrawlIssues';
|
|
147
|
+
interface BingCallOptions {
|
|
148
|
+
signal?: AbortSignal;
|
|
149
|
+
}
|
|
150
|
+
type BingCrawlDateFilter = 'any' | 'last-week' | 'last-two-weeks' | 'last-three-weeks';
|
|
151
|
+
type BingDiscoveredDateFilter = 'any' | 'last-week' | 'last-month';
|
|
152
|
+
type BingDocumentFilter = 'any' | 'blocked-by-robots-txt' | 'malware';
|
|
153
|
+
type BingHttpCodeFilter = 'any' | '2xx' | '3xx' | '301' | '302' | '4xx' | '5xx' | 'other';
|
|
154
|
+
interface BingChildrenFilters {
|
|
155
|
+
crawlDate?: BingCrawlDateFilter;
|
|
156
|
+
discoveredDate?: BingDiscoveredDateFilter;
|
|
157
|
+
document?: BingDocumentFilter;
|
|
158
|
+
httpCode?: BingHttpCodeFilter;
|
|
159
|
+
}
|
|
160
|
+
interface BingChildrenOptions extends BingCallOptions {
|
|
161
|
+
filters?: BingChildrenFilters;
|
|
162
|
+
maxPages?: number;
|
|
163
|
+
}
|
|
164
|
+
interface BingWebmasterClient {
|
|
165
|
+
getUserSites: (options?: BingCallOptions) => Promise<Result<BingSite[], BingProviderError>>;
|
|
166
|
+
getVerifiedSite: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingSite, BingProviderError>>;
|
|
167
|
+
getUrlInfo: (siteUrl: string, url: string, options?: BingCallOptions) => Promise<Result<BingUrlInfo | null, BingProviderError>>;
|
|
168
|
+
getIndexingEvidence: (siteUrl: string, url: string, options?: BingCallOptions) => Promise<Result<BingIndexingEvidence, BingProviderError | BingEvidenceError>>;
|
|
169
|
+
getUrlTrafficInfo: (siteUrl: string, url: string, options?: BingCallOptions) => Promise<Result<BingUrlTrafficInfo | null, BingProviderError>>;
|
|
170
|
+
getChildrenUrlInfo: (siteUrl: string, url: string, options?: BingChildrenOptions) => Promise<Result<BingUrlInfo[], BingProviderError>>;
|
|
171
|
+
getPageStats: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingPageStats[], BingProviderError>>;
|
|
172
|
+
getCrawlIssues: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingUrlWithCrawlIssues[], BingProviderError>>;
|
|
173
|
+
}
|
|
174
|
+
type BingFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
175
|
+
interface BingWebmasterOptions {
|
|
176
|
+
accessToken: string;
|
|
177
|
+
baseUrl?: string;
|
|
178
|
+
clock?: () => Date;
|
|
179
|
+
fetch?: BingFetch;
|
|
180
|
+
}
|
|
181
|
+
export { BingCallOptions, BingChildrenFilters, BingChildrenOptions, BingCrawlDateFilter, BingCrawlEvidence, BingCrawlIssue, BingCrawlIssueWire, BingDiscoveredDateFilter, BingDocumentFilter, BingEvidenceError, BingFetch, BingHttpCodeFilter, BingIndexingEvidence, BingOperation, BingPageStats, BingProviderError, BingQueryStatsWire, BingSite, BingSiteWire, BingUnknownEvidence, BingUrlInfo, BingUrlInfoWire, BingUrlTrafficInfo, BingUrlTrafficInfoWire, BingUrlWithCrawlIssues, BingWebmasterClient, BingWebmasterOptions, BingWireResponse };
|
package/dist/query/plan.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { TableName } from "../contracts.mjs";
|
|
2
1
|
import { Result } from "../core/result.mjs";
|
|
2
|
+
import { TableName } from "../contracts.mjs";
|
|
3
3
|
import { BuilderState, Dimension, FilterOperator, Metric, MetricOperator, QueryParamName } from "./types.mjs";
|
|
4
4
|
import { QueryError, QueryErrorKind, UnresolvableDatasetError, UnsupportedLogicalCapabilityError } from "./errors.mjs";
|
|
5
5
|
type LogicalDataset = TableName;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GscSearchAnalyticsRequest } from "../contracts.mjs";
|
|
2
1
|
import { Result } from "../core/result.mjs";
|
|
2
|
+
import { GscSearchAnalyticsRequest } from "../contracts.mjs";
|
|
3
3
|
import { SearchType } from "./constants.mjs";
|
|
4
4
|
import { BuilderState, Filter, FilterInput, InternalFilter } from "./types.mjs";
|
|
5
5
|
import { QueryError } from "./errors.mjs";
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gscdump",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "2.
|
|
5
|
-
"description": "Direct Google Search Console
|
|
4
|
+
"version": "2.2.0",
|
|
5
|
+
"description": "Direct Google Search Console and Bing Webmaster clients with typed queries and Indexing Evidence",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
},
|
|
22
22
|
"keywords": [
|
|
23
23
|
"google-search-console",
|
|
24
|
+
"bing-webmaster-tools",
|
|
24
25
|
"gsc",
|
|
25
26
|
"seo",
|
|
26
27
|
"search-analytics",
|
|
@@ -42,6 +43,11 @@
|
|
|
42
43
|
"import": "./dist/query/index.mjs",
|
|
43
44
|
"default": "./dist/query/index.mjs"
|
|
44
45
|
},
|
|
46
|
+
"./bing": {
|
|
47
|
+
"types": "./dist/bing/index.d.mts",
|
|
48
|
+
"import": "./dist/bing/index.mjs",
|
|
49
|
+
"default": "./dist/bing/index.mjs"
|
|
50
|
+
},
|
|
45
51
|
"./query/plan": {
|
|
46
52
|
"types": "./dist/query/plan.d.mts",
|
|
47
53
|
"import": "./dist/query/plan.mjs",
|
|
@@ -88,7 +94,7 @@
|
|
|
88
94
|
},
|
|
89
95
|
"dependencies": {
|
|
90
96
|
"ofetch": "^1.5.1",
|
|
91
|
-
"@gscdump/contracts": "^2.
|
|
97
|
+
"@gscdump/contracts": "^2.2.0"
|
|
92
98
|
},
|
|
93
99
|
"scripts": {
|
|
94
100
|
"dev": "obuild --stub",
|